db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-10-30 23:37:45 +02:00
Code Issues Releases Activity

Add/remove dependent backups during backup.info reconstruct.

Browse Source 
Prior to performing a backup or expiring backups, the backup.info file is validated by reconstructing it from the backups in the repository. When a backup had already been removed from the repo, it was removed from the backup.info file but its dependents were not.

Now, the dependent backups will also be removed from backup.info and only backups in the repo that have their full dependency chain will be added to backup.info if they are missing.
This commit is contained in:
Cynthia Shang
2020-04-03 13:25:38 -04:00
committed by David Steele
parent f3ae74b0d6
commit 3fbde30c6f
2 changed files with 113 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
src/info/infoBackup.c
View File

@@ -694,9 +694,34 @@ infoBackupLoadFileReconstruct(const Storage *storage, const String *fileName, Ci
.expression = backupRegExpP(.full = true, .differential = true, .incremental = true)),
sortOrderAsc);
// Get the current list of backups from backup.info
// Get the list of current backups and remove backups from current that are no longer in the repository
StringList *backupCurrentList = strLstSort(infoBackupDataLabelList(infoBackup, NULL), sortOrderAsc);
for (unsigned int backupCurrIdx = 0; backupCurrIdx < strLstSize(backupCurrentList); backupCurrIdx++)
{
String *backupLabel = strLstGet(backupCurrentList, backupCurrIdx);
String *manifestFileName = strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strPtr(backupLabel));
// If the manifest does not exist on disk and this backup has not already been deleted from the current list in the
// infoBackup object, then remove it and its dependencies
if (!storageExistsP(storage, manifestFileName) && infoBackupDataByLabel(infoBackup, backupLabel) != NULL)
{
StringList *backupList = strLstSort(infoBackupDataDependentList(infoBackup, backupLabel), sortOrderDesc);
for (unsigned int backupIdx = 0; backupIdx < strLstSize(backupList); backupIdx++)
{
String *removeBackup = strLstGet(backupList, backupIdx);
LOG_WARN_FMT("backup '%s' missing manifest removed from " INFO_BACKUP_FILE, strPtr(removeBackup));
infoBackupDataDelete(infoBackup, removeBackup);
}
}
}
// Get the updated current list of backups from backup.info
backupCurrentList = strLstSort(infoBackupDataLabelList(infoBackup, NULL), sortOrderAsc);
// For each backup in the repo, check if it exists in backup.info
for (unsigned int backupIdx = 0; backupIdx < strLstSize(backupList); backupIdx++)
{
@@ -721,9 +746,12 @@ infoBackupLoadFileReconstruct(const Storage *storage, const String *fileName, Ci
{
InfoPgData pgHistory = infoPgData(infoBackup->infoPg, pgIdx);
// If there is an exact match with the history, system and version then add it to the current backup list
// If there is an exact match with the history, system and version and there is no backup-prior dependency
// or there is a backup-prior and it is in the list, then add this backup to the current backup list
if (manData->pgId == pgHistory.id && manData->pgSystemId == pgHistory.systemId &&
manData->pgVersion == pgHistory.version)
manData->pgVersion == pgHistory.version &&
(manData->backupLabelPrior == NULL ||
infoBackupDataByLabel(infoBackup, manData->backupLabelPrior) != NULL))
{
LOG_WARN_FMT("backup '%s' found in repository added to " INFO_BACKUP_FILE, strPtr(backupLabel));
infoBackupDataAdd(infoBackup, manifest);
@@ -737,22 +765,6 @@ infoBackupLoadFileReconstruct(const Storage *storage, const String *fileName, Ci
}
}
}
// Get the updated list of current backups and remove backups from current that are no longer in the repository
backupCurrentList = infoBackupDataLabelList(infoBackup, NULL);
for (unsigned int backupCurrIdx = 0; backupCurrIdx < strLstSize(backupCurrentList); backupCurrIdx++)
{
String *backupLabel = strLstGet(backupCurrentList, backupCurrIdx);
String *manifestFileName = strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strPtr(backupLabel));
// Remove backup from the current list in the infoBackup object
if (!storageExistsP(storage, manifestFileName))
{
LOG_WARN_FMT("backup '%s' missing manifest removed from " INFO_BACKUP_FILE, strPtr(backupLabel));
infoBackupDataDelete(infoBackup, backupLabel);
}
}
}
MEM_CONTEXT_TEMP_END();

93
test/src/module/info/infoBackupTest.c
View File

@@ -335,7 +335,7 @@ testRun(void)
"template0={\"db-id\":12168,\"db-last-system-id\":12168}\n" \
"template1={\"db-id\":1,\"db-last-system-id\":12168}\n" \
manifestContent = harnessInfoChecksumZ
const Buffer *manifestContentIncr = harnessInfoChecksumZ
(
"[backup]\n"
"backup-archive-start=\"000000030000028500000089\"\n"
@@ -405,7 +405,7 @@ testRun(void)
TEST_MANIFEST_PATH_DEFAULT
);
TEST_ASSIGN(manifest, manifestNewLoad(ioBufferReadNew(manifestContent)), "load manifest");
TEST_ASSIGN(manifest, manifestNewLoad(ioBufferReadNew(manifestContentIncr)), "load manifest");
TEST_RESULT_VOID(infoBackupDataAdd(infoBackup, manifest), "add a backup");
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 2, "backup added to current");
TEST_ASSIGN(backupData, infoBackupData(infoBackup, 1), "get added backup");
@@ -668,6 +668,11 @@ testRun(void)
strNew(STORAGE_REPO_BACKUP "/20190818-084777F/" BACKUP_MANIFEST_FILE)),
manifestContent), "write manifest - invalid backup version mismatch");
TEST_RESULT_VOID(
storagePutP(storageNewWriteP(storageRepoWrite(),
strNew(STORAGE_REPO_BACKUP "/20190818-084502F_20190820-084502I/" BACKUP_MANIFEST_FILE)),
manifestContentIncr), "write manifest for dependent backup that will be removed from backup.info");
TEST_RESULT_VOID(
storagePathCreateP(storageRepoWrite(), strNew(STORAGE_REPO_BACKUP "/20190818-084502F")),
"create backup on disk that is in current but no manifest");
@@ -680,8 +685,8 @@ testRun(void)
.expression = backupRegExpP(.full = true, .differential = true, .incremental = true)),
sortOrderAsc),
", "),
"20190818-084444F, 20190818-084502F, 20190818-084555F, 20190818-084666F, 20190818-084777F, 20190923-164324F",
"confirm backups on disk");
"20190818-084444F, 20190818-084502F, 20190818-084502F_20190820-084502I, 20190818-084555F, 20190818-084666F, "
"20190818-084777F, 20190923-164324F", "confirm backups on disk");
// With the infoBackup from above, upgrade the DB so there a 2 histories then save to disk
TEST_ASSIGN(infoBackup, infoBackupPgSet(infoBackup, PG_VERSION_11, 6739907367085689196), "upgrade db");
@@ -701,34 +706,100 @@ testRun(void)
TEST_RESULT_INT(infoBackupDataTotal(infoBackup), 1, "backup list contains 1 backup");
TEST_ASSIGN(backupData, infoBackupData(infoBackup, 0), "get the backup");
TEST_RESULT_STR_Z(
backupData.backupLabel, "20190923-164324F",
"backups not on disk removed, valid backup on disk added, manifest copy-only ignored");
backupData.backupLabel, "20190923-164324F", "backups not on disk removed, dependent backup removed and not added back, "
"valid backup on disk added, manifest copy-only ignored");
harnessLogResult(
"P00 WARN: backup '20190818-084502F_20190820-084502I' missing manifest removed from backup.info\n"
"P00 WARN: backup '20190818-084502F' missing manifest removed from backup.info\n"
"P00 WARN: invalid backup '20190818-084502F_20190820-084502I' cannot be added to current backups\n"
"P00 WARN: invalid backup '20190818-084555F' cannot be added to current backups\n"
"P00 WARN: invalid backup '20190818-084666F' cannot be added to current backups\n"
"P00 WARN: invalid backup '20190818-084777F' cannot be added to current backups\n"
"P00 WARN: backup '20190923-164324F' found in repository added to backup.info\n"
"P00 WARN: backup '20190818-084502F' missing manifest removed from backup.info\n"
"P00 WARN: backup '20190818-084502F_20190820-084502I' missing manifest removed from backup.info");
"P00 WARN: backup '20190923-164324F' found in repository added to backup.info");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(
storagePathRemoveP(storageRepoWrite(), strNew(STORAGE_REPO_BACKUP "/20190818-084502F_20190820-084502I"),
.recurse = true), "remove dependent backup from disk");
TEST_ASSIGN(
infoBackup, infoBackupLoadFileReconstruct(storageRepo(), INFO_BACKUP_PATH_FILE_STR, cipherTypeNone, NULL),
"reconstruct does not attempt to add back dependent backup");
harnessLogResult(
"P00 WARN: backup '20190818-084502F_20190820-084502I' missing manifest removed from backup.info\n"
"P00 WARN: backup '20190818-084502F' missing manifest removed from backup.info\n"
"P00 WARN: invalid backup '20190818-084555F' cannot be added to current backups\n"
"P00 WARN: invalid backup '20190818-084666F' cannot be added to current backups\n"
"P00 WARN: invalid backup '20190818-084777F' cannot be added to current backups\n"
"P00 WARN: backup '20190923-164324F' found in repository added to backup.info");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(
storageCopyP(
storageNewReadP(storageRepo(), strNew(STORAGE_REPO_BACKUP "/20190818-084444F/" BACKUP_MANIFEST_FILE INFO_COPY_EXT)),
storageNewWriteP(storageRepoWrite(), strNew(STORAGE_REPO_BACKUP "/20190818-084444F/" BACKUP_MANIFEST_FILE))),
"write manifest from copy-only for pgId=1");
manifestContentIncr = harnessInfoChecksumZ
(
"[backup]\n"
"backup-archive-start=\"000000030000028500000089\"\n"
"backup-archive-stop=\"000000030000028500000090\"\n"
"backup-label=\"20190818-084444F_20190924-084502D\"\n"
"backup-lsn-start=\"285/89000028\"\n"
"backup-lsn-stop=\"285/89001F88\"\n"
"backup-prior=\"20190818-084444F\"\n"
"backup-timestamp-copy-start=1565282141\n"
"backup-timestamp-start=1565282140\n"
"backup-timestamp-stop=1565282142\n"
"backup-type=\"diff\"\n"
TEST_MANIFEST_BACKUPDB
"\n"
"[backup:option]\n"
"option-archive-check=true\n"
"option-archive-copy=true\n"
"option-backup-standby=true\n"
"option-buffer-size=16384\n"
"option-checksum-page=true\n"
"option-compress=true\n"
"option-compress-level=3\n"
"option-compress-level-network=3\n"
"option-delta=false\n"
"option-hardlink=true\n"
"option-online=true\n"
"option-process-max=32\n"
"\n"
"[backup:target]\n"
"pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n"
"\n"
"[target:file]\n"
"pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"size\":4,\"timestamp\":1565282114}\n"
TEST_MANIFEST_FILE_DEFAULT
"\n"
"[target:path]\n"
"pg_data={}\n"
TEST_MANIFEST_PATH_DEFAULT
);
TEST_RESULT_VOID(
storagePutP(storageNewWriteP(storageRepoWrite(),
strNew(STORAGE_REPO_BACKUP "/20190818-084444F_20190924-084502D/" BACKUP_MANIFEST_FILE)),
manifestContentIncr), "write manifest for dependent backup to be added to full already in backup.info");
TEST_RESULT_VOID(
infoBackupSaveFile(infoBackup, storageRepoWrite(), INFO_BACKUP_PATH_FILE_STR, cipherTypeNone, NULL),
"save updated backup info");
infoBackup = NULL;
TEST_ASSIGN(
infoBackup, infoBackupLoadFileReconstruct(storageRepo(), INFO_BACKUP_PATH_FILE_STR, cipherTypeNone, NULL),
"reconstruct");
TEST_RESULT_STR_Z(
strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", "), "20190818-084444F, 20190923-164324F",
"previously ignored pgId=1 manifest copy-only now added before existing");
strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", "),
"20190818-084444F, 20190818-084444F_20190924-084502D, 20190923-164324F",
"previously ignored pgId=1 manifest copy-only now added before existing, and add dependent found");
harnessLogResult(
"P00 WARN: backup '20190818-084444F' found in repository added to backup.info\n"
"P00 WARN: backup '20190818-084444F_20190924-084502D' found in repository added to backup.info\n"
"P00 WARN: invalid backup '20190818-084555F' cannot be added to current backups\n"
"P00 WARN: invalid backup '20190818-084666F' cannot be added to current backups\n"
"P00 WARN: invalid backup '20190818-084777F' cannot be added to current backups");