db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-04-13 11:30:40 +02:00
Code Issues Releases Activity

Enhance expire command multi-repo support.

Browse Source 
The expire command has been enhanced to expire backups and archives from all configured repositories by default.

In addition, it will accept the --repo option to expire backups and archives only from the specified repository. Using the --repo options the --set option can also be refined further to the specified repo. If --set is provided but the --repo option has not, then all repositories will be searched and retention settings will be applied on each whether the backup set has been found or not.
This commit is contained in:
Cynthia Shang 2021-02-10 12:03:52 -05:00 committed by David Steele
parent 26cbebbda7
commit 3408f1ee2e
7 changed files with 404 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
doc/xml/release.xml
View File

@ -13,6 +13,19 @@
<release-list>
<release date="XXXX-XX-XX" version="2.33dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-development-list>
<release-item>
<commit subject="Enhance expire command multi-repo support."/>
<release-item-contributor-list>
<release-item-contributor id="cynthia.shang"/>
</release-item-contributor-list>
<p>Partial multi-repository implementation.</p>
</release-item>
</release-development-list>
</release-core-list>
</release>
<release date="2021-02-08" version="2.32" title="Repository Commands">

2
src/command/backup/backup.c
View File

@ -1904,7 +1904,7 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest)
// Create a symlink to the most recent backup if supported. This link is purely informational for the user and is never
// used by us since symlinks are not supported on all storage types.
// -------------------------------------------------------------------------------------------------------------------------
backupLinkLatest(backupLabel);
backupLinkLatest(backupLabel, cfgOptionGroupIdxDefault(cfgOptGrpRepo));
// Add manifest and save backup.info (infoBackupSaveFile() is responsible for proper syncing)
// -------------------------------------------------------------------------------------------------------------------------

13
src/command/backup/common.c
View File

@ -139,10 +139,11 @@ const String *backupTypeStr(BackupType type)
/**********************************************************************************************************************************/
void
backupLinkLatest(const String *backupLabel)
backupLinkLatest(const String *backupLabel, unsigned int repoIdx)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, backupLabel);
FUNCTION_TEST_PARAM(UINT, repoIdx);
FUNCTION_TEST_END();
MEM_CONTEXT_TEMP_BEGIN()
@ -150,12 +151,12 @@ backupLinkLatest(const String *backupLabel)
// Create a symlink to the most recent backup if supported. This link is purely informational for the user and is never
// used by us since symlinks are not supported on all storage types.
// -------------------------------------------------------------------------------------------------------------------------
const String *const latestLink = storagePathP(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/" BACKUP_LINK_LATEST));
const String *const latestLink = storagePathP(storageRepoIdx(repoIdx), STRDEF(STORAGE_REPO_BACKUP "/" BACKUP_LINK_LATEST));
// Remove an existing latest link/file in case symlink capabilities have changed
storageRemoveP(storageRepoWrite(), latestLink);
storageRemoveP(storageRepoIdxWrite(repoIdx), latestLink);
if (storageFeature(storageRepoWrite(), storageFeatureSymLink))
if (storageFeature(storageRepoIdxWrite(repoIdx), storageFeatureSymLink))
{
THROW_ON_SYS_ERROR_FMT(
symlink(strZ(backupLabel), strZ(latestLink)) == -1, FileOpenError, "unable to create symlink '%s' to '%s'",
@ -163,8 +164,8 @@ backupLinkLatest(const String *backupLabel)
}
// Sync backup path if required
if (storageFeature(storageRepoWrite(), storageFeaturePathSync))
storagePathSyncP(storageRepoWrite(), STORAGE_REPO_BACKUP_STR);
if (storageFeature(storageRepoIdxWrite(repoIdx), storageFeaturePathSync))
storagePathSyncP(storageRepoIdxWrite(repoIdx), STORAGE_REPO_BACKUP_STR);
}
MEM_CONTEXT_TEMP_END();

2
src/command/backup/common.h
View File

@ -52,6 +52,6 @@ BackupType backupType(const String *type);
const String *backupTypeStr(BackupType type);
// Create a symlink to the specified backup (if symlinks are supported)
void backupLinkLatest(const String *backupLabel);
void backupLinkLatest(const String *backupLabel, unsigned int repoIdx);
#endif

275
src/command/expire/expire.c
View File

@ -39,11 +39,12 @@ typedef struct ArchiveRange
Given a backup label, expire a backup and all its dependents (if any).
***********************************************************************************************************************************/
static StringList *
expireBackup(InfoBackup *infoBackup, const String *backupLabel)
expireBackup(InfoBackup *infoBackup, const String *backupLabel, unsigned int repoIdx)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup);
FUNCTION_LOG_PARAM(STRING, backupLabel);
FUNCTION_TEST_PARAM(UINT, repoIdx);
FUNCTION_LOG_END();
ASSERT(infoBackup != NULL);
@ -68,9 +69,10 @@ expireBackup(InfoBackup *infoBackup, const String *backupLabel)
{
// Remove the manifest files to invalidate the backup
storageRemoveP(
storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(removeBackupLabel)));
storageRepoIdxWrite(repoIdx),
strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(removeBackupLabel)));
storageRemoveP(
storageRepoWrite(),
storageRepoIdxWrite(repoIdx),
strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(removeBackupLabel)));
}
@ -106,73 +108,58 @@ expireAdhocBackup(InfoBackup *infoBackup, const String *backupLabel, unsigned in
MEM_CONTEXT_TEMP_BEGIN()
{
// If the label format is invalid, then error
if (!regExpMatchOne(backupRegExpP(.full = true, .differential = true, .incremental = true), backupLabel))
// Get a list of all full backups with most recent in position 0
StringList *fullList = strLstSort(infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true)), sortOrderDesc);
// If the requested backup to expire is the latest full backup
if (strCmp(strLstGet(fullList, 0), backupLabel) == 0)
{
THROW_FMT(OptionInvalidValueError, "'%s' is not a valid backup label format", strZ(backupLabel));
// If the latest full backup requested is the only backup or the prior full backup is not for the same db-id
// then the backup requested cannot be expired
if (strLstSize(fullList) == 1 || infoBackupDataByLabel(infoBackup, backupLabel)->backupPgId !=
infoBackupDataByLabel(infoBackup, strLstGet(fullList, 1))->backupPgId)
{
THROW_FMT(
BackupSetInvalidError,
"full backup repo%u: %s cannot be expired until another full backup has been created on this repo",
cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx), strZ(backupLabel));
}
}
// If the label is not a current backup then notify user and exit
if (infoBackupDataByLabel(infoBackup, backupLabel) == NULL)
// Save off what is currently the latest backup (it may be removed if it is the adhoc backup or is a dependent of the
// adhoc backup
const String *latestBackup = infoBackupData(infoBackup, infoBackupDataTotal(infoBackup) - 1).backupLabel;
// Expire the requested backup and any dependents
StringList *backupExpired = expireBackup(infoBackup, backupLabel, repoIdx);
// If the latest backup was removed, then update the latest link if not a dry-run
if (infoBackupDataByLabel(infoBackup, latestBackup) == NULL)
{
// If retention settings have been configured, then there may be holes in the archives. For example, if the archive
// for db-id=1 has 01,02,03,04,05 and F1 backup has archive start-stop 02-03 and rentention-full=1
// (hence retention-archive=1 and retention-archive-type=full), then when F2 backup is created and assuming its
// archive start-stop=05-06 then archives 01 and 04 will be removed resulting in F1 not being able to play through
// PITR, which is expected. Now adhoc expire is attempted on F2 - it will be allowed but now there will be no
// backups that can be recovered through PITR until the next full backup is created. Same problem for differential
// backups with retention-diff.
LOG_WARN_FMT(
"backup %s does not exist\nHINT: run the info command and confirm the backup is listed", strZ(backupLabel));
"expiring latest backup repo%u: %s - the ability to perform point-in-time-recovery (PITR) may be affected\n"
"HINT: non-default settings for '%s'/'%s' (even in prior expires) can cause gaps in the WAL.",
cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx), strZ(latestBackup),
cfgOptionIdxName(cfgOptRepoRetentionArchive, repoIdx), cfgOptionIdxName(cfgOptRepoRetentionArchiveType, repoIdx));
// Adhoc expire is never performed through backup command so only check to determine if dry-run has been set or not
if (!cfgOptionBool(cfgOptDryRun))
backupLinkLatest(infoBackupData(infoBackup, infoBackupDataTotal(infoBackup) - 1).backupLabel, repoIdx);
}
else
{
// Get a list of all full backups with most recent in position 0
StringList *fullList = strLstSort(infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true)), sortOrderDesc);
// If the requested backup to expire is the latest full backup
if (strCmp(strLstGet(fullList, 0), backupLabel) == 0)
{
// If the latest full backup requested is the only backup or the prior full backup is not for the same db-id
// then the backup requested cannot be expired
if (strLstSize(fullList) == 1 || infoBackupDataByLabel(infoBackup, backupLabel)->backupPgId !=
infoBackupDataByLabel(infoBackup, strLstGet(fullList, 1))->backupPgId)
{
THROW_FMT(
BackupSetInvalidError,
"full backup repo%u: %s cannot be expired until another full backup has been created on this repo",
cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx), strZ(backupLabel));
}
}
result = strLstSize(backupExpired);
// Save off what is currently the latest backup (it may be removed if it is the adhoc backup or is a dependent of the
// adhoc backup
const String *latestBackup = infoBackupData(infoBackup, infoBackupDataTotal(infoBackup) - 1).backupLabel;
// Expire the requested backup and any dependents
StringList *backupExpired = expireBackup(infoBackup, backupLabel);
// If the latest backup was removed, then update the latest link if not a dry-run
if (infoBackupDataByLabel(infoBackup, latestBackup) == NULL)
{
// If retention settings have been configured, then there may be holes in the archives. For example, if the archive
// for db-id=1 has 01,02,03,04,05 and F1 backup has archive start-stop 02-03 and rentention-full=1
// (hence retention-archive=1 and retention-archive-type=full), then when F2 backup is created and assuming its
// archive start-stop=05-06 then archives 01 and 04 will be removed resulting in F1 not being able to play through
// PITR, which is expected. Now adhoc expire is attempted on F2 - it will be allowed but now there will be no
// backups that can be recovered through PITR until the next full backup is created. Same problem for differential
// backups with retention-diff.
LOG_WARN_FMT(
"expiring latest backup repo%u: %s - the ability to perform point-in-time-recovery (PITR) may be affected\n"
"HINT: non-default settings for '%s'/'%s' (even in prior expires) can cause gaps in the WAL.",
cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx), strZ(latestBackup), cfgOptionName(cfgOptRepoRetentionArchive),
cfgOptionName(cfgOptRepoRetentionArchiveType));
// Adhoc expire is never performed through backup command so only check to determine if dry-run has been set or not
if (!cfgOptionBool(cfgOptDryRun))
backupLinkLatest(infoBackupData(infoBackup, infoBackupDataTotal(infoBackup) - 1).backupLabel);
}
result = strLstSize(backupExpired);
// Log the expired backup list (prepend "set:" if there were any dependents that were also expired)
LOG_INFO_FMT(
"expire adhoc backup %srepo%u: %s", (result > 1 ? "set " : ""), cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx),
strZ(strLstJoin(backupExpired, ", ")));
}
// Log the expired backup list (prepend "set:" if there were any dependents that were also expired)
LOG_INFO_FMT(
"expire adhoc backup %srepo%u: %s", (result > 1 ? "set " : ""), cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx),
strZ(strLstJoin(backupExpired, ", ")));
}
MEM_CONTEXT_TEMP_END();
@ -197,7 +184,8 @@ expireDiffBackup(InfoBackup *infoBackup, unsigned int repoIdx)
MEM_CONTEXT_TEMP_BEGIN()
{
unsigned int differentialRetention = cfgOptionTest(cfgOptRepoRetentionDiff) ? cfgOptionUInt(cfgOptRepoRetentionDiff) : 0;
unsigned int differentialRetention = cfgOptionIdxTest(
cfgOptRepoRetentionDiff, repoIdx) ? cfgOptionIdxUInt(cfgOptRepoRetentionDiff, repoIdx) : 0;
// Find all the expired differential backups
if (differentialRetention > 0)
@ -217,7 +205,7 @@ expireDiffBackup(InfoBackup *infoBackup, unsigned int repoIdx)
continue;
// Expire the differential and any dependent backups
StringList *backupExpired = expireBackup(infoBackup, strLstGet(currentBackupList, diffIdx));
StringList *backupExpired = expireBackup(infoBackup, strLstGet(currentBackupList, diffIdx), repoIdx);
result += strLstSize(backupExpired);
// Log the expired backups. If there is more than one backup, then prepend "set:"
@ -251,7 +239,8 @@ expireFullBackup(InfoBackup *infoBackup, unsigned int repoIdx)
MEM_CONTEXT_TEMP_BEGIN()
{
unsigned int fullRetention = cfgOptionTest(cfgOptRepoRetentionFull) ? cfgOptionUInt(cfgOptRepoRetentionFull) : 0;
unsigned int fullRetention = cfgOptionIdxTest(
cfgOptRepoRetentionFull, repoIdx) ? cfgOptionIdxUInt(cfgOptRepoRetentionFull, repoIdx) : 0;
// Find all the expired full backups
if (fullRetention > 0)
@ -266,7 +255,7 @@ expireFullBackup(InfoBackup *infoBackup, unsigned int repoIdx)
for (unsigned int fullIdx = 0; fullIdx < strLstSize(currentBackupList) - fullRetention; fullIdx++)
{
// Expire the full backup and all its dependents
StringList *backupExpired = expireBackup(infoBackup, strLstGet(currentBackupList, fullIdx));
StringList *backupExpired = expireBackup(infoBackup, strLstGet(currentBackupList, fullIdx), repoIdx);
result += strLstSize(backupExpired);
// Log the expired backups. If there is more than one backup, then prepend "set:"
@ -333,7 +322,7 @@ expireTimeBasedBackup(InfoBackup *infoBackup, const time_t minTimestamp, unsigne
// always the oldest so if it is not the backup to keep then we can remove it
while (!strEq(infoBackupData(infoBackup, 0).backupLabel, lastBackupLabelToKeep))
{
StringList *backupExpired = expireBackup(infoBackup, infoBackupData(infoBackup, 0).backupLabel);
StringList *backupExpired = expireBackup(infoBackup, infoBackupData(infoBackup, 0).backupLabel, repoIdx);
result += strLstSize(backupExpired);
numFullExpired++;
@ -344,11 +333,12 @@ expireTimeBasedBackup(InfoBackup *infoBackup, const time_t minTimestamp, unsigne
cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx), strZ(strLstJoin(backupExpired, ", ")));
}
if (strEqZ(cfgOptionStr(cfgOptRepoRetentionArchiveType), CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_FULL) &&
!cfgOptionTest(cfgOptRepoRetentionArchive) && numFullExpired > 0)
if (strEqZ(cfgOptionIdxStr(cfgOptRepoRetentionArchiveType, repoIdx), CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_FULL) &&
!cfgOptionIdxTest(cfgOptRepoRetentionArchive, repoIdx) && numFullExpired > 0)
{
cfgOptionSet(
cfgOptRepoRetentionArchive, cfgSourceDefault, varNewUInt(strLstSize(currentBackupList) - numFullExpired));
cfgOptionIdxSet(
cfgOptRepoRetentionArchive, repoIdx, cfgSourceDefault,
varNewUInt(strLstSize(currentBackupList) - numFullExpired));
}
}
}
@ -391,8 +381,9 @@ removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention, unsign
MEM_CONTEXT_TEMP_BEGIN()
{
// Get the retention options. repo-archive-retention-type always has a value as it defaults to "full"
const String *archiveRetentionType = cfgOptionStr(cfgOptRepoRetentionArchiveType);
unsigned int archiveRetention = cfgOptionTest(cfgOptRepoRetentionArchive) ? cfgOptionUInt(cfgOptRepoRetentionArchive) : 0;
const String *archiveRetentionType = cfgOptionIdxStr(cfgOptRepoRetentionArchiveType, repoIdx);
unsigned int archiveRetention = cfgOptionIdxTest(
cfgOptRepoRetentionArchive, repoIdx) ? cfgOptionIdxUInt(cfgOptRepoRetentionArchive, repoIdx) : 0;
// If archive retention is undefined, then ignore archiving. The user does not have to set this - it will be defaulted in
// cfgLoadUpdateOption based on certain rules.
@ -402,7 +393,7 @@ removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention, unsign
// Only notify user if not time-based retention
if (!timeBasedFullRetention)
LOG_INFO_FMT("option '%s' is not set %s", cfgOptionName(cfgOptRepoRetentionArchive), strZ(msg));
LOG_INFO_FMT("option '%s' is not set %s", cfgOptionIdxName(cfgOptRepoRetentionArchive, repoIdx), strZ(msg));
else
{
LOG_INFO_FMT(
@ -439,15 +430,16 @@ removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention, unsign
{
// Attempt to load the archive info file
InfoArchive *infoArchive = infoArchiveLoadFile(
storageRepo(), INFO_ARCHIVE_PATH_FILE_STR, cipherType(cfgOptionStr(cfgOptRepoCipherType)),
cfgOptionStrNull(cfgOptRepoCipherPass));
storageRepoIdx(repoIdx), INFO_ARCHIVE_PATH_FILE_STR, cipherType(cfgOptionIdxStr(cfgOptRepoCipherType, repoIdx)),
cfgOptionIdxStrNull(cfgOptRepoCipherPass, repoIdx));
InfoPg *infoArchivePgData = infoArchivePg(infoArchive);
// Get a list of archive directories (e.g. 9.4-1, 10-2, etc) sorted by the db-id (number after the dash).
StringList *listArchiveDisk = strLstSort(
strLstComparatorSet(
storageListP(storageRepo(), STORAGE_REPO_ARCHIVE_STR, .expression = STRDEF(REGEX_ARCHIVE_DIR_DB_VERSION)),
storageListP(
storageRepoIdx(repoIdx), STORAGE_REPO_ARCHIVE_STR, .expression = STRDEF(REGEX_ARCHIVE_DIR_DB_VERSION)),
archiveIdComparator),
sortOrderAsc);
@ -531,7 +523,7 @@ removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention, unsign
if (currentPg.id != archivePgId)
{
String *fullPath = storagePathP(
storageRepo(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s", strZ(archiveId)));
storageRepoIdx(repoIdx), strNewFmt(STORAGE_REPO_ARCHIVE "/%s", strZ(archiveId)));
LOG_INFO_FMT(
"remove archive path repo%u: %s", cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx),
@ -539,7 +531,7 @@ removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention, unsign
// Execute the real expiration and deletion only if the dry-run option is disabled
if (!cfgOptionValid(cfgOptDryRun) || !cfgOptionBool(cfgOptDryRun))
storagePathRemoveP(storageRepoWrite(), fullPath, .recurse = true);
storagePathRemoveP(storageRepoIdxWrite(repoIdx), fullPath, .recurse = true);
}
// Continue to next directory
@ -640,7 +632,7 @@ removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention, unsign
StringList *walPathList =
strLstSort(
storageListP(
storageRepo(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s", strZ(archiveId)),
storageRepoIdx(repoIdx), strNewFmt(STORAGE_REPO_ARCHIVE "/%s", strZ(archiveId)),
.expression = STRDEF(WAL_SEGMENT_DIR_REGEXP)),
sortOrderAsc);
@ -669,7 +661,7 @@ removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention, unsign
if (!cfgOptionValid(cfgOptDryRun) || !cfgOptionBool(cfgOptDryRun))
{
storagePathRemoveP(
storageRepoWrite(),
storageRepoIdxWrite(repoIdx),
strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strZ(archiveId), strZ(walPath)),
.recurse = true);
}
@ -687,7 +679,7 @@ removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention, unsign
StringList *walSubPathList =
strLstSort(
storageListP(
storageRepo(),
storageRepoIdx(repoIdx),
strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strZ(archiveId), strZ(walPath)),
.expression = STRDEF("^[0-F]{24}.*$")),
sortOrderAsc);
@ -718,7 +710,7 @@ removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention, unsign
if (!cfgOptionValid(cfgOptDryRun) || !cfgOptionBool(cfgOptDryRun))
{
storageRemoveP(
storageRepoWrite(),
storageRepoIdxWrite(repoIdx),
strNewFmt(
STORAGE_REPO_ARCHIVE "/%s/%s/%s", strZ(archiveId), strZ(walPath),
strZ(walSubPath)));
@ -753,7 +745,7 @@ removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention, unsign
StringList *historyFilesList =
strLstSort(
storageListP(
storageRepo(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s", strZ(archiveId)),
storageRepoIdx(repoIdx), strNewFmt(STORAGE_REPO_ARCHIVE "/%s", strZ(archiveId)),
.expression = WAL_TIMELINE_HISTORY_REGEXP_STR),
sortOrderAsc);
@ -769,7 +761,7 @@ removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention, unsign
if (!cfgOptionValid(cfgOptDryRun) || !cfgOptionBool(cfgOptDryRun))
{
storageRemoveP(
storageRepoWrite(),
storageRepoIdxWrite(repoIdx),
strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strZ(archiveId), strZ(historyFile)));
}
@ -810,7 +802,7 @@ removeExpiredBackup(InfoBackup *infoBackup, const String *adhocBackupLabel, unsi
// Get all the backups on disk
StringList *backupList = strLstSort(
storageListP(
storageRepo(), STORAGE_REPO_BACKUP_STR,
storageRepoIdx(repoIdx), STORAGE_REPO_BACKUP_STR,
.expression = backupRegExpP(.full = true, .differential = true, .incremental = true)),
sortOrderDesc);
@ -825,7 +817,8 @@ removeExpiredBackup(InfoBackup *infoBackup, const String *adhocBackupLabel, unsi
String *manifestCopyFileName = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFileName));
// If the latest backup is resumable (has a backup.manifest.copy but no backup.manifest)
if (!storageExistsP(storageRepo(), manifestFileName) && storageExistsP(storageRepo(), manifestCopyFileName))
if (!storageExistsP(storageRepoIdx(repoIdx), manifestFileName) &&
storageExistsP(storageRepoIdx(repoIdx), manifestCopyFileName))
{
// If the resumable backup is not related to the expired adhoc backup then don't remove it
if (!strBeginsWith(strLstGet(backupList, backupIdx), strSubN(adhocBackupLabel, 0, 16)))
@ -836,7 +829,7 @@ removeExpiredBackup(InfoBackup *infoBackup, const String *adhocBackupLabel, unsi
else
{
Manifest *manifestResume = manifestLoadFile(
storageRepo(), manifestFileName, cipherType(cfgOptionStr(cfgOptRepoCipherType)),
storageRepoIdx(repoIdx), manifestFileName, cipherType(cfgOptionIdxStr(cfgOptRepoCipherType, repoIdx)),
infoPgCipherPass(infoBackupPg(infoBackup)));
// If the ancestor of the resumable backup still exists in backup.info then do not remove the resumable backup
@ -859,7 +852,7 @@ removeExpiredBackup(InfoBackup *infoBackup, const String *adhocBackupLabel, unsi
if (!cfgOptionValid(cfgOptDryRun) || !cfgOptionBool(cfgOptDryRun))
{
storagePathRemoveP(
storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s", strZ(strLstGet(backupList, backupIdx))),
storageRepoIdxWrite(repoIdx), strNewFmt(STORAGE_REPO_BACKUP "/%s", strZ(strLstGet(backupList, backupIdx))),
.recurse = true);
}
}
@ -882,53 +875,93 @@ cmdExpire(void)
MEM_CONTEXT_TEMP_BEGIN()
{
// Load the backup.info
InfoBackup *infoBackup = infoBackupLoadFileReconstruct(
storageRepo(), INFO_BACKUP_PATH_FILE_STR, cipherType(cfgOptionStr(cfgOptRepoCipherType)),
cfgOptionStrNull(cfgOptRepoCipherPass));
// Initialize the repo index
unsigned int repoIdxMin = 0;
unsigned int repoIdxMax = cfgOptionGroupIdxTotal(cfgOptGrpRepo) - 1;
// If the repo was specified then set index to the array location and max to loop only once
if (cfgOptionTest(cfgOptRepo))
{
repoIdxMin = cfgOptionGroupIdxDefault(cfgOptGrpRepo);
repoIdxMax = repoIdxMin;
}
// Get the backup label if specified
const String *adhocBackupLabel = NULL;
bool timeBasedFullRetention = strEqZ(
cfgOptionStr(cfgOptRepoRetentionFullType), CFGOPTVAL_TMP_REPO_RETENTION_FULL_TYPE_TIME);
bool adhocBackupFound = false;
unsigned int repoIdx = cfgOptionGroupIdxDefault(cfgOptGrpRepo);
// If the --set option is valid (i.e. expire is called on its own) and is set then attempt to expire the requested backup
// If the --set option is valid (i.e. expire is called on its own) then check the label format
if (cfgOptionTest(cfgOptSet))
{
adhocBackupLabel = cfgOptionStr(cfgOptSet);
expireAdhocBackup(infoBackup, adhocBackupLabel, repoIdx);
// If the label format is invalid, then error
if (!regExpMatchOne(backupRegExpP(.full = true, .differential = true, .incremental = true), adhocBackupLabel))
THROW_FMT(OptionInvalidValueError, "'%s' is not a valid backup label format", strZ(adhocBackupLabel));
}
else
for (unsigned int repoIdx = repoIdxMin; repoIdx <= repoIdxMax; repoIdx++)
{
// If time-based retention for full backups is set, then expire based on time period
if (timeBasedFullRetention)
// Get the repo storage in case it is remote and encryption settings need to be pulled down
const Storage *storageRepo = storageRepoIdx(repoIdx);
// Load the backup.info
InfoBackup *infoBackup = infoBackupLoadFileReconstruct(
storageRepo, INFO_BACKUP_PATH_FILE_STR, cipherType(cfgOptionIdxStr(cfgOptRepoCipherType, repoIdx)),
cfgOptionIdxStrNull(cfgOptRepoCipherPass, repoIdx));
bool timeBasedFullRetention = strEqZ(
cfgOptionIdxStr(cfgOptRepoRetentionFullType, repoIdx), CFGOPTVAL_TMP_REPO_RETENTION_FULL_TYPE_TIME);
// If a backupLabel was set, then attempt to expire the requested backup
if (adhocBackupLabel != NULL)
{
// If a time period was provided then run time-based expiration otherwise do nothing (the user has already been
// warned by the config system that retention-full was not set)
if (cfgOptionTest(cfgOptRepoRetentionFull))
if (infoBackupDataByLabel(infoBackup, adhocBackupLabel) != NULL)
{
expireTimeBasedBackup(
infoBackup, time(NULL) - (time_t)(cfgOptionUInt(cfgOptRepoRetentionFull) * SEC_PER_DAY), repoIdx);
adhocBackupFound = true;
expireAdhocBackup(infoBackup, adhocBackupLabel, repoIdx);
}
// If the adhoc backup was not found and this was the last repo to check, then log a warning but continue to process
// the expiration based on retention
if (!adhocBackupFound && repoIdx == repoIdxMax)
{
LOG_WARN_FMT(
"backup %s does not exist\nHINT: run the info command and confirm the backup is listed",
strZ(adhocBackupLabel));
}
}
else
expireFullBackup(infoBackup, repoIdx);
{
// If time-based retention for full backups is set, then expire based on time period
if (timeBasedFullRetention)
{
// If a time period was provided then run time-based expiration otherwise do nothing (the user has already been
// warned by the config system that retention-full was not set)
if (cfgOptionIdxTest(cfgOptRepoRetentionFull, repoIdx))
{
expireTimeBasedBackup(
infoBackup, time(NULL) - (time_t)(cfgOptionUInt(cfgOptRepoRetentionFull) * SEC_PER_DAY), repoIdx);
}
}
else
expireFullBackup(infoBackup, repoIdx);
expireDiffBackup(infoBackup, repoIdx);
expireDiffBackup(infoBackup, repoIdx);
}
// Store the new backup info only if the dry-run mode is disabled
if (!cfgOptionValid(cfgOptDryRun) || !cfgOptionBool(cfgOptDryRun))
{
infoBackupSaveFile(
infoBackup, storageRepoIdxWrite(repoIdx), INFO_BACKUP_PATH_FILE_STR,
cipherType(cfgOptionIdxStr(cfgOptRepoCipherType, repoIdx)), cfgOptionIdxStrNull(cfgOptRepoCipherPass, repoIdx));
}
// Remove all files on disk that are now expired
removeExpiredBackup(infoBackup, adhocBackupLabel, repoIdx);
removeExpiredArchive(infoBackup, timeBasedFullRetention, repoIdx);
}
// Store the new backup info only if the dry-run mode is disabled
if (!cfgOptionValid(cfgOptDryRun) || !cfgOptionBool(cfgOptDryRun))
{
infoBackupSaveFile(
infoBackup, storageRepoWrite(), INFO_BACKUP_PATH_FILE_STR, cipherType(cfgOptionStr(cfgOptRepoCipherType)),
cfgOptionStrNull(cfgOptRepoCipherPass));
}
// Remove all files on disk that are now expired
removeExpiredBackup(infoBackup, adhocBackupLabel, repoIdx);
removeExpiredArchive(infoBackup, timeBasedFullRetention, repoIdx);
}
MEM_CONTEXT_TEMP_END();

4
src/config/load.c
View File

@ -70,8 +70,8 @@ cfgLoadUpdateOption(void)
// Make sure repo option is set for the default command role when it is not internal and more than one repo is configured or the
// first configured repo is not key 1. Filter out any commands where this does not apply.
if (!cfgCommandHelp() && cfgCommand() != cfgCmdInfo && cfgOptionValid(cfgOptRepo) && !cfgOptionTest(cfgOptRepo) &&
(cfgOptionGroupIdxTotal(cfgOptGrpRepo) > 1 || cfgOptionGroupIdxToKey(cfgOptGrpRepo, 0) != 1))
if (!cfgCommandHelp() && cfgCommand() != cfgCmdInfo && cfgCommand() != cfgCmdExpire && cfgOptionValid(cfgOptRepo) &&
!cfgOptionTest(cfgOptRepo) && (cfgOptionGroupIdxTotal(cfgOptGrpRepo) > 1 || cfgOptionGroupIdxToKey(cfgOptGrpRepo, 0) != 1))
{
THROW_FMT(
OptionRequiredError,

316
test/src/module/command/expireTest.c
View File

@ -190,13 +190,12 @@ testRun(void)
BUFSTRDEF(BOGUS_STR)), "full1 put extra file");
TEST_RESULT_VOID(storagePathCreateP(storageTest, full2Path), "full2 empty");
TEST_RESULT_VOID(expireBackup(infoBackup, full1), "expire backup with both manifest files");
TEST_RESULT_VOID(expireBackup(infoBackup, full1, 0), "expire backup with both manifest files");
TEST_RESULT_BOOL(
(strLstSize(storageListP(storageTest, full1Path)) && strLstExistsZ(storageListP(storageTest, full1Path), "bogus")),
true, "full1 - only manifest files removed");
TEST_RESULT_VOID(expireBackup(infoBackup, full2), "expire backup with no manifest - does not error");
TEST_RESULT_VOID(expireBackup(infoBackup, full2, 0), "expire backup with no manifest - does not error");
TEST_RESULT_STRLST_Z(
infoBackupDataLabelList(infoBackup, NULL),
@ -868,9 +867,11 @@ testRun(void)
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionDiff, 2, "3");
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionArchive, 2, "2");
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionArchiveType, 2, "diff");
hrnCfgArgRawZ(argList, cfgOptRepo, "2");
strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
harnessCfgLoad(cfgCmdBackup, argList);
StringList *argList2 = strLstDup(argList);
hrnCfgArgRawZ(argList2, cfgOptRepo, "2");
strLstAdd(argList2, strNewFmt("--pg1-path=%s/pg", testPath()));
harnessCfgLoad(cfgCmdBackup, argList2);
TEST_RESULT_VOID(cmdExpire(), "via backup command: expire last backup in archive sub path and remove sub path");
TEST_RESULT_BOOL(
@ -892,25 +893,10 @@ testRun(void)
"P00 INFO: expire full backup repo2: 20181119-152138F\n"
"P00 INFO: remove expired backup repo2: 20181119-152138F");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("expire command requires repo option");
argList = strLstDup(argListBase);
hrnCfgArgKeyRawFmt(argList, cfgOptRepoPath, 2, "%s/repo2", testPath());
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionFull, 2, "3");
TEST_ERROR_FMT(
harnessCfgLoad(cfgCmdExpire, argList), OptionRequiredError, "expire command requires option: repo\n"
"HINT: this command requires a specific repository to operate on");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("expire command - no dry run");
// Add to previous list and specify repo
strLstAddZ(argList, "--repo1-retention-full=2");
strLstAddZ(argList, "--repo1-retention-diff=3");
strLstAddZ(argList, "--repo1-retention-archive=2");
strLstAddZ(argList, "--repo1-retention-archive-type=diff");
hrnCfgArgRawZ(argList, cfgOptRepo, "1");
harnessCfgLoad(cfgCmdExpire, argList);
@ -923,12 +909,21 @@ testRun(void)
"P00 INFO: remove expired backup repo1: 20181119-152138F");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("expire command - dry run: archive and backups not removed");
TEST_TITLE("expire command - multi-repo, dry run: archive and backups not removed");
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-archive=1");
strLstAddZ(argList, "--dry-run");
harnessCfgLoad(cfgCmdExpire, argList);
hrnCfgArgKeyRawFmt(argList, cfgOptRepoPath, 2, "%s/repo2", testPath());
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionFull, 2, "3");
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionDiff, 2, "2");
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionArchive, 2, "1");
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionArchiveType, 2, "diff");
argList2 = strLstDup(argList);
strLstAddZ(argList2, "--dry-run");
harnessCfgLoad(cfgCmdExpire, argList2);
harnessLogLevelSet(logLevelDetail);
TEST_RESULT_VOID(cmdExpire(), "expire (dry-run) - log expired backups and archive path to remove");
TEST_RESULT_BOOL(
@ -943,6 +938,7 @@ testRun(void)
storageExistsP(
storageTest, strNewFmt("%s/20181119-152800F_20181119-152252D/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath)))),
true, "backup not removed");
harnessLogResult(strZ(strNewFmt(
"P00 INFO: [DRY-RUN] expire full backup set repo1: 20181119-152800F, 20181119-152800F_20181119-152152D, "
"20181119-152800F_20181119-152155I, 20181119-152800F_20181119-152252D\n"
@ -950,17 +946,29 @@ testRun(void)
"P00 INFO: [DRY-RUN] remove expired backup repo1: 20181119-152800F_20181119-152155I\n"
"P00 INFO: [DRY-RUN] remove expired backup repo1: 20181119-152800F_20181119-152152D\n"
"P00 INFO: [DRY-RUN] remove expired backup repo1: 20181119-152800F\n"
"P00 INFO: [DRY-RUN] remove archive path repo1: %s/%s/9.4-1", testPath(), strZ(archiveStanzaPath))));
"P00 INFO: [DRY-RUN] remove archive path repo1: %s/%s/9.4-1\n"
"P00 DETAIL: [DRY-RUN] archive retention on backup 20181119-152900F repo1: 10-2, start = 000000010000000000000003\n"
"P00 DETAIL: [DRY-RUN] no archive to remove for repo1: 10-2\n"
"P00 INFO: [DRY-RUN] expire diff backup set repo2: 20181119-152800F_20181119-152152D,"
" 20181119-152800F_20181119-152155I\n"
"P00 INFO: [DRY-RUN] remove expired backup repo2: 20181119-152800F_20181119-152155I\n"
"P00 INFO: [DRY-RUN] remove expired backup repo2: 20181119-152800F_20181119-152152D\n"
"P00 DETAIL: [DRY-RUN] archive retention on backup 20181119-152800F repo2: 9.4-1, start = 000000020000000000000002,"
" stop = 000000020000000000000002\n"
"P00 DETAIL: [DRY-RUN] archive retention on backup 20181119-152800F_20181119-152252D repo2: 9.4-1,"
" start = 000000020000000000000009\n"
"P00 DETAIL: [DRY-RUN] remove archive repo2: 9.4-1, start = 000000020000000000000004,"
" stop = 000000020000000000000007\n"
"P00 DETAIL: [DRY-RUN] archive retention on backup 20181119-152900F repo2: 10-2, start = 000000010000000000000003\n"
"P00 DETAIL: [DRY-RUN] no archive to remove for repo2: 10-2", testPath(), strZ(archiveStanzaPath))));
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("expire via backup command - archive and backups removed");
TEST_TITLE("expire command - multi-repo, archive and backups removed");
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-archive=1");
strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
harnessCfgLoad(cfgCmdBackup, argList);
// Rerun previous test without dry-run
harnessCfgLoad(cfgCmdExpire, argList);
TEST_RESULT_VOID(cmdExpire(), "via backup command: expire backups and remove archive path");
TEST_RESULT_VOID(cmdExpire(), "expire backups and remove archive path");
TEST_RESULT_BOOL(
storagePathExistsP(storageTest, strNewFmt("%s/%s", strZ(archiveStanzaPath), "9.4-1")),
false, "archive path removed");
@ -972,7 +980,21 @@ testRun(void)
"P00 INFO: remove expired backup repo1: 20181119-152800F_20181119-152155I\n"
"P00 INFO: remove expired backup repo1: 20181119-152800F_20181119-152152D\n"
"P00 INFO: remove expired backup repo1: 20181119-152800F\n"
"P00 INFO: remove archive path repo1: %s/%s/9.4-1", testPath(), strZ(archiveStanzaPath))));
"P00 INFO: remove archive path repo1: %s/%s/9.4-1\n"
"P00 DETAIL: archive retention on backup 20181119-152900F repo1: 10-2, start = 000000010000000000000003\n"
"P00 DETAIL: no archive to remove for repo1: 10-2\n"
"P00 INFO: expire diff backup set repo2: 20181119-152800F_20181119-152152D,"
" 20181119-152800F_20181119-152155I\n"
"P00 INFO: remove expired backup repo2: 20181119-152800F_20181119-152155I\n"
"P00 INFO: remove expired backup repo2: 20181119-152800F_20181119-152152D\n"
"P00 DETAIL: archive retention on backup 20181119-152800F repo2: 9.4-1, start = 000000020000000000000002,"
" stop = 000000020000000000000002\n"
"P00 DETAIL: archive retention on backup 20181119-152800F_20181119-152252D repo2: 9.4-1,"
" start = 000000020000000000000009\n"
"P00 DETAIL: remove archive repo2: 9.4-1, start = 000000020000000000000004,"
" stop = 000000020000000000000007\n"
"P00 DETAIL: archive retention on backup 20181119-152900F repo2: 10-2, start = 000000010000000000000003\n"
"P00 DETAIL: no archive to remove for repo2: 10-2", testPath(), strZ(archiveStanzaPath))));
TEST_ASSIGN(infoBackup, infoBackupLoadFile(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info");
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 2, "backup.info updated on disk");
@ -980,13 +1002,60 @@ testRun(void)
strLstSort(infoBackupDataLabelList(infoBackup, NULL), sortOrderAsc),
"20181119-152900F\n20181119-152900F_20181119-152500I\n", "remaining current backups correct");
harnessLogLevelReset();
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("expire command - multi-repo, adhoc");
// With multi-repo config from previous test, adhoc expire on backup that doesn't exist
argList2 = strLstDup(argList);
hrnCfgArgRawZ(argList2, cfgOptSet, "20201119-123456F_20201119-234567I");
harnessCfgLoad(cfgCmdExpire, argList2);
TEST_RESULT_VOID(cmdExpire(), "label format OK but backup does not exist on any repo");
harnessLogResult(
"P00 WARN: backup 20201119-123456F_20201119-234567I does not exist\n"
" HINT: run the info command and confirm the backup is listed");
// Rerun on single repo
hrnCfgArgRawZ(argList2, cfgOptRepo, "1");
harnessCfgLoad(cfgCmdExpire, argList2);
TEST_RESULT_VOID(cmdExpire(), "label format OK but backup does not exist on requested repo");
harnessLogResult(
"P00 WARN: backup 20201119-123456F_20201119-234567I does not exist\n"
" HINT: run the info command and confirm the backup is listed");
// With multiple repos, adhoc expire backup only on one repo
hrnCfgArgRawZ(argList, cfgOptSet, "20181119-152900F_20181119-152500I");
hrnCfgArgRawZ(argList, cfgOptRepo, "1");
hrnCfgArgRawBool(argList, cfgOptDryRun, true);
harnessCfgLoad(cfgCmdExpire, argList);
TEST_RESULT_VOID(cmdExpire(), "label format OK and expired on specified repo");
harnessLogResult(
"P00 WARN: [DRY-RUN] expiring latest backup repo1: 20181119-152900F_20181119-152500I - the ability to perform"
" point-in-time-recovery (PITR) may be affected\n"
" HINT: non-default settings for 'repo1-retention-archive'/'repo1-retention-archive-type' (even in prior"
" expires) can cause gaps in the WAL.\n"
"P00 INFO: [DRY-RUN] expire adhoc backup repo1: 20181119-152900F_20181119-152500I\n"
"P00 INFO: [DRY-RUN] remove expired backup repo1: 20181119-152900F_20181119-152500I");
// Incorrect backup label format provided
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--set=" BOGUS_STR);
harnessCfgLoad(cfgCmdExpire, argList);
TEST_ERROR(cmdExpire(), OptionInvalidValueError, "'" BOGUS_STR "' is not a valid backup label format");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("expire command - archive removed");
archiveGenerate(storageTest, archiveStanzaPath, 1, 1, "9.4-1", "0000000100000000");
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-archive=1");
harnessCfgLoad(cfgCmdExpire, argList);
strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
harnessCfgLoad(cfgCmdBackup, argList);
TEST_RESULT_VOID(cmdExpire(), "expire remove archive path");
harnessLogResult(
@ -1618,20 +1687,6 @@ testRun(void)
archiveGenerate(storageTest, archiveStanzaPath, 1, 10, "9.4-1", "0000000200000000");
archiveGenerate(storageTest, archiveStanzaPath, 1, 10, "12-2", "0000000100000000");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("invalid backup label");
TEST_RESULT_UINT(
expireAdhocBackup(infoBackup, STRDEF("20201119-123456F_20201119-234567I"), 0), 0,
"label format OK but backup does not exist");
harnessLogResult(
"P00 WARN: backup 20201119-123456F_20201119-234567I does not exist\n"
" HINT: run the info command and confirm the backup is listed");
TEST_ERROR(
expireAdhocBackup(infoBackup, STRDEF(BOGUS_STR), 0), OptionInvalidValueError,
"'" BOGUS_STR "' is not a valid backup label format");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("expire backup and dependent");
@ -1864,54 +1919,61 @@ testRun(void)
"P00 INFO: [DRY-RUN] remove expired backup repo1: 20181119-152850F_20181119-152252D");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("resumable possibly based on adhoc expire backup");
TEST_TITLE("resumable possibly based on adhoc expire backup, multi-repo, encryption");
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--set=20181119-152850F_20181119-152252D");
hrnCfgArgRawZ(argList, cfgOptSet, "20181119-152850F_20181119-152252D");
hrnCfgArgKeyRawFmt(argList, cfgOptRepoPath, 2, "%s/repo2", testPath());
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionFull, 2, "1");
hrnCfgArgKeyRawZ(argList, cfgOptRepoCipherType, 2, CIPHER_TYPE_AES_256_CBC);
hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 2, TEST_CIPHER_PASS);
harnessCfgLoad(cfgCmdExpire, argList);
// Create backup.info
storagePutP(storageNewWriteP(storageTest, backupInfoFileName),
harnessInfoChecksumZ(
"[backup:current]\n"
"20181119-152850F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000004\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":2,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152850F_20181119-152252D={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000006\","
"\"backup-archive-stop\":\"000000010000000000000007\",\"backup-info-repo-size\":2369186,"
"\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428,"
"\"backup-prior\":\"20181119-152850F\",\"backup-reference\":[\"20181119-152850F\"],"
"\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\","
"\"db-id\":2,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"\n"
"[db]\n"
"db-catalog-version=201909212\n"
"db-control-version=1201\n"
"db-id=2\n"
"db-system-id=6626363367545678089\n"
"db-version=\"12\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702,"
"\"db-version\":\"9.4\"}\n"
"2={\"db-catalog-version\":201909212,\"db-control-version\":1201,\"db-system-id\":6626363367545678089,"
"\"db-version\":\"12\"}\n"));
#define TEST_BACKUP_CURRENT \
"[backup:current]\n" \
"20181119-152850F={" \
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," \
"\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000004\"," \
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," \
"\"db-id\":2,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" \
"20181119-152850F_20181119-152252D={" \
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000006\"," \
"\"backup-archive-stop\":\"000000010000000000000007\",\"backup-info-repo-size\":2369186," \
"\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428," \
"\"backup-prior\":\"20181119-152850F\",\"backup-reference\":[\"20181119-152850F\"]," \
"\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\"," \
"\"db-id\":2,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":false}\n"
#define TEST_BACKUP_DB \
"\n" \
"[db]\n" \
"db-catalog-version=201909212\n" \
"db-control-version=1201\n" \
"db-id=2\n" \
"db-system-id=6626363367545678089\n" \
"db-version=\"12\"\n" \
"\n" \
"[db:history]\n" \
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," \
"\"db-version\":\"9.4\"}\n" \
"2={\"db-catalog-version\":201909212,\"db-control-version\":1201,\"db-system-id\":6626363367545678089," \
"\"db-version\":\"12\"}\n"
String *backupInfoContent = strNew(
TEST_BACKUP_CURRENT
TEST_BACKUP_DB);
storagePutP(storageNewWriteP(storageTest, backupInfoFileName), harnessInfoChecksum(backupInfoContent));
storagePutP(
storageNewWriteP(storageTest, strNewFmt("%s" INFO_COPY_EXT, strZ(backupInfoFileName))),
harnessInfoChecksum(backupInfoContent));
// Adhoc backup and resumable backup manifests
storagePutP(
storageNewWriteP(storageTest, strNewFmt("%s/20181119-152850F_20181119-152252D/" BACKUP_MANIFEST_FILE,
strZ(backupStanzaPath))), BUFSTRDEF("tmp"));
storagePutP(
storageNewWriteP(storageTest, strNewFmt("%s/20181119-152850F_20181200-152252D/" BACKUP_MANIFEST_FILE INFO_COPY_EXT,
strZ(backupStanzaPath))),
harnessInfoChecksumZ(
String *manifestContent = strNew(
"[backup]\n"
"backup-archive-start=\"000000010000000000000009\"\n"
"backup-label=null\n"
@ -1958,10 +2020,71 @@ testRun(void)
"[target:path:default]\n"
"group=\"postgres\"\n"
"mode=\"0700\"\n"
"user=\"postgres\"\n"));
"user=\"postgres\"\n");
storagePutP(
storageNewWriteP(storageTest, strNewFmt("%s/20181119-152850F_20181119-152252D/" BACKUP_MANIFEST_FILE,
strZ(backupStanzaPath))), BUFSTRDEF("tmp"));
storagePutP(
storageNewWriteP(storageTest, strNewFmt("%s/20181119-152850F_20181200-152252D/" BACKUP_MANIFEST_FILE INFO_COPY_EXT,
strZ(backupStanzaPath))), harnessInfoChecksum(manifestContent));
// archives to repo1
archiveGenerate(storageTest, archiveStanzaPath, 2, 10, "12-2", "0000000100000000");
// Create encrypted repo2 with same data from repo1 and ensure results are reported the same. This will test that the
// manifest can be read on encrypted repos.
String *repo2ArchiveStanzaPath = strNewFmt("%s/repo2/archive/db", testPath());
String *repo2BackupStanzaPath = strNewFmt("%s/repo2/backup/db", testPath());
storagePathCreateP(storageLocalWrite(), repo2ArchiveStanzaPath);
storagePathCreateP(storageLocalWrite(), repo2BackupStanzaPath);
HRN_INFO_PUT(
storageTest, strZ(strNewFmt("%s/archive.info", strZ(repo2ArchiveStanzaPath))),
"[cipher]\n"
"cipher-pass=\"" TEST_CIPHER_PASS_ARCHIVE "\"\n"
"\n"
"[db]\n"
"db-id=2\n"
"db-system-id=6626363367545678089\n"
"db-version=\"12\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":6625592122879095702,\"db-version\":\"9.4\"}\n"
"2={\"db-id\":6626363367545678089,\"db-version\":\"12\"}",
.cipherType = cipherTypeAes256Cbc);
backupInfoContent = strNew(
TEST_BACKUP_CURRENT
"\n"
"[cipher]\n"
"cipher-pass=\"somepass\"\n"
TEST_BACKUP_DB);
String *repo2BackupInfoFileName = strNewFmt("%s/backup.info", strZ(repo2BackupStanzaPath));
HRN_INFO_PUT(storageTest, strZ(repo2BackupInfoFileName), strZ(backupInfoContent), .cipherType = cipherTypeAes256Cbc);
HRN_INFO_PUT(
storageTest, strZ(strNewFmt("%s" INFO_COPY_EXT, strZ(repo2BackupInfoFileName))), strZ(backupInfoContent),
.cipherType = cipherTypeAes256Cbc);
HRN_INFO_PUT(
storageTest, strZ(strNewFmt("%s/20181119-152850F/" BACKUP_MANIFEST_FILE, strZ(repo2BackupStanzaPath))),
"[backup]\nbackup-type=\"full\"\n", .cipherType = cipherTypeAes256Cbc, .cipherPass = "somepass");
HRN_INFO_PUT(
storageTest, strZ(strNewFmt("%s/20181119-152850F_20181119-152252D/" BACKUP_MANIFEST_FILE, strZ(repo2BackupStanzaPath))),
"[backup]\nbackup-type=\"diff\"\n", .cipherType = cipherTypeAes256Cbc, .cipherPass = "somepass");
HRN_INFO_PUT(
storageTest, strZ(strNewFmt("%s/20181119-152850F_20181200-152252D/" BACKUP_MANIFEST_FILE INFO_COPY_EXT,
strZ(repo2BackupStanzaPath))), strZ(manifestContent), .cipherType = cipherTypeAes256Cbc, .cipherPass = "somepass");
// archives to repo2
archiveGenerate(storageTest, repo2ArchiveStanzaPath, 2, 10, "12-2", "0000000100000000");
// Create "latest" symlink, repo2
latestLink = storagePathP(storageTest, strNewFmt("%s/latest", strZ(repo2BackupStanzaPath)));
THROW_ON_SYS_ERROR_FMT(
symlink("20181119-152850F_20181200-152252D", strZ(latestLink)) == -1,
FileOpenError, "unable to create symlink '%s' to '%s'", strZ(latestLink), "20181119-152850F_20181200-152252D");
TEST_RESULT_VOID(cmdExpire(), "adhoc expire latest with resumable possibly based on it");
harnessLogResult(
"P00 WARN: expiring latest backup repo1: 20181119-152850F_20181119-152252D - the ability to perform"
@ -1971,8 +2094,21 @@ testRun(void)
"P00 INFO: expire adhoc backup repo1: 20181119-152850F_20181119-152252D\n"
"P00 INFO: remove expired backup repo1: 20181119-152850F_20181119-152252D\n"
"P00 DETAIL: archive retention on backup 20181119-152850F repo1: 12-2, start = 000000010000000000000002\n"
"P00 DETAIL: no archive to remove for repo1: 12-2");
"P00 DETAIL: no archive to remove for repo1: 12-2\n"
"P00 WARN: expiring latest backup repo2: 20181119-152850F_20181119-152252D - the ability to perform"
" point-in-time-recovery (PITR) may be affected\n"
" HINT: non-default settings for 'repo2-retention-archive'/'repo2-retention-archive-type'"
" (even in prior expires) can cause gaps in the WAL.\n"
"P00 INFO: expire adhoc backup repo2: 20181119-152850F_20181119-152252D\n"
"P00 INFO: remove expired backup repo2: 20181119-152850F_20181119-152252D\n"
"P00 DETAIL: archive retention on backup 20181119-152850F repo2: 12-2, start = 000000010000000000000002\n"
"P00 DETAIL: no archive to remove for repo2: 12-2");
TEST_RESULT_STR(storageInfoP(storageRepoIdx(1), STRDEF(STORAGE_REPO_BACKUP "/latest")).linkDestination,
STRDEF("20181119-152850F"), "latest link updated, repo2");
// Cleanup
hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 2);
harnessLogLevelReset();
}