db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-11-06 08:49:29 +02:00
Code Issues Releases Activity

Add persistent reference list to manifest.

Browse Source 
The reference list was previously built at load time from whichever references existed in the file list. This was sufficient since the list was for informational purposes only.

The block incremental feature will require a reference list that contains all prior backups, even those that are not explicitly referenced from the manifest. Therefore it makes sense to build and persist a manifest list rather than building it at load time.

This list can still be used for informational purposes, though it needs to be sorted since the list it sill built for older manifest versions and may not be in sorted order.

Add strLstFindIdx() to find references in the list.
This commit is contained in:
David Steele
2022-10-05 16:28:31 -10:00
committed by GitHub
parent c647bcb509
commit 102ce5dee4
12 changed files with 114 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
doc/xml/release.xml
View File

@@ -54,10 +54,14 @@
<commit subject="Add manifest flags for file processing during backup.">
<github-pull-request id="1889"/>
</commit>
<commit subject="Add persistent reference list to manifest.">
<github-pull-request id="1890"/>
</commit>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="stefan.fercot"/>
<release-item-reviewer id="reid.thompson"/>
<release-item-reviewer id="john.morris"/>
</release-item-contributor-list>

21
src/common/type/stringList.c
View File

@@ -274,6 +274,27 @@ strLstAddZSubN(StringList *const this, const char *const string, const size_t of
FUNCTION_TEST_RETURN(STRING, result);
}
/**********************************************************************************************************************************/
unsigned int
strLstFindIdx(const StringList *const this, const String *const string, const StrLstFindIdxParam param)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING_LIST, this);
FUNCTION_TEST_PARAM(STRING, string);
FUNCTION_TEST_PARAM(BOOL, param.required);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(string != NULL);
const unsigned int result = lstFindIdx((List *)this, &string);
if (result == LIST_NOT_FOUND && param.required)
THROW_FMT(AssertError, "unable to find '%s' in string list", strZ(string));
FUNCTION_TEST_RETURN(UINT, result);
}
/**********************************************************************************************************************************/
String *
strLstInsert(StringList *this, unsigned int listIdx, const String *string)

12
src/common/type/stringList.h
View File

@@ -95,6 +95,18 @@ strLstExists(const StringList *const this, const String *const string)
return lstExists((List *)this, &string);
}
// Find string index in the list
typedef struct StrLstFindIdxParam
{
VAR_PARAM_HEADER;
bool required;
} StrLstFindIdxParam;
#define strLstFindIdxP(this, string, ...) \
strLstFindIdx(this, string, (StrLstFindIdxParam){VAR_PARAM_INIT, __VA_ARGS__})
unsigned int strLstFindIdx(const StringList *this, const String *string, StrLstFindIdxParam param);
// Insert into the list
String *strLstInsert(StringList *this, unsigned int listIdx, const String *string);

9
src/info/infoBackup.c
View File

@@ -391,7 +391,6 @@ infoBackupDataAdd(const InfoBackup *this, const Manifest *manifest)
uint64_t backupSizeDelta = 0;
uint64_t backupRepoSize = 0;
uint64_t backupRepoSizeDelta = 0;
StringList *referenceList = strLstNew();
bool backupError = false;
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
@@ -402,9 +401,7 @@ infoBackupDataAdd(const InfoBackup *this, const Manifest *manifest)
backupRepoSize += file.sizeRepo > 0 ? file.sizeRepo : file.size;
// If a reference to a file exists, then it is in a previous backup and the delta calculation was already done
if (file.reference != NULL)
strLstAddIfMissing(referenceList, file.reference);
else
if (file.reference == NULL)
{
backupSizeDelta += file.size;
backupRepoSizeDelta += file.sizeRepo > 0 ? file.sizeRepo : file.size;
@@ -450,8 +447,8 @@ infoBackupDataAdd(const InfoBackup *this, const Manifest *manifest)
if (manData->backupType != backupTypeFull)
{
strLstSort(referenceList, sortOrderAsc);
infoBackupData.backupReference = strLstDup(referenceList);
// This list may not be sorted for manifests created before the reference list was added
infoBackupData.backupReference = strLstSort(strLstDup(manifestReferenceList(manifest)), sortOrderAsc);
infoBackupData.backupPrior = strDup(manData->backupLabelPrior);
}

29
src/info/manifest.c
View File

@@ -34,7 +34,6 @@ struct Manifest
{
ManifestPub pub; // Publicly accessible variables
StringList *ownerList; // List of users/groups
StringList *referenceList; // List of file references
const String *fileUserDefault; // Default file user name
const String *fileGroupDefault; // Default file group name
@@ -186,7 +185,7 @@ manifestFilePack(const Manifest *const manifest, const ManifestFile *const file)
if (file->reference != NULL)
{
cvtUInt64ToVarInt128(
(uintptr_t)strLstAddIfMissing(manifest->referenceList, file->reference), buffer, &bufferPos, sizeof(buffer));
strLstFindIdxP(manifest->pub.referenceList, file->reference, .required = true), buffer, &bufferPos, sizeof(buffer));
}
// Mode
@@ -283,7 +282,10 @@ manifestFileUnpack(const Manifest *const manifest, const ManifestFilePack *const
// Reference
if (flag & (1 << manifestFilePackFlagReference))
result.reference = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX);
{
result.reference = strLstGet(
manifest->pub.referenceList, (unsigned int)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX));
}
// Mode
if (flag & (1 << manifestFilePackFlagMode))
@@ -481,9 +483,9 @@ manifestNewInternal(void)
.linkList = lstNewP(sizeof(ManifestLink), .comparator = lstComparatorStr),
.pathList = lstNewP(sizeof(ManifestPath), .comparator = lstComparatorStr),
.targetList = lstNewP(sizeof(ManifestTarget), .comparator = lstComparatorStr),
.referenceList = strLstNew(),
},
.ownerList = strLstNew(),
.referenceList = strLstNew(),
};
FUNCTION_TEST_RETURN(MANIFEST, this);
@@ -1426,6 +1428,9 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
// Set prior backup label
this->pub.data.backupLabelPrior = strDup(manifestPrior->pub.data.backupLabel);
// Copy reference list
this->pub.referenceList = strLstDup(manifestPrior->pub.referenceList);
// Set diff/incr backup type
this->pub.data.backupType = type;
}
@@ -1665,6 +1670,7 @@ manifestBuildComplete(
#define MANIFEST_KEY_BACKUP_LSN_START "backup-lsn-start"
#define MANIFEST_KEY_BACKUP_LSN_STOP "backup-lsn-stop"
#define MANIFEST_KEY_BACKUP_PRIOR "backup-prior"
#define MANIFEST_KEY_BACKUP_REFERENCE "backup-reference"
#define MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START "backup-timestamp-copy-start"
#define MANIFEST_KEY_BACKUP_TIMESTAMP_START "backup-timestamp-start"
#define MANIFEST_KEY_BACKUP_TIMESTAMP_STOP "backup-timestamp-stop"
@@ -1720,6 +1726,7 @@ typedef struct ManifestLoadData
{
MemContext *memContext; // Mem context for data needed only during load
Manifest *manifest; // Manifest info
bool referenceListFound; // Was a reference list found?
List *linkFoundList; // Values found in links
const Variant *linkGroupDefault; // Link default group
@@ -1837,8 +1844,13 @@ manifestLoadCallback(void *callbackData, const String *const section, const Stri
// Reference
if (jsonReadKeyExpectStrId(json, MANIFEST_KEY_REFERENCE))
{
file.reference = jsonReadStr(json);
if (!loadData->referenceListFound)
file.reference = strLstAddIfMissing(manifest->pub.referenceList, file.reference);
}
// If "repo-size" is not present in the manifest file, then it is the same as size (i.e. uncompressed) - to save space,
// the repo-size is only stored in the manifest file if it is different than size.
const bool sizeRepoExists = jsonReadKeyExpectStrId(json, MANIFEST_KEY_SIZE_REPO);
@@ -2059,6 +2071,11 @@ manifestLoadCallback(void *callbackData, const String *const section, const Stri
manifest->pub.data.lsnStop = varStr(jsonToVar(value));
else if (strEqZ(key, MANIFEST_KEY_BACKUP_PRIOR))
manifest->pub.data.backupLabelPrior = varStr(jsonToVar(value));
else if (strEqZ(key, MANIFEST_KEY_BACKUP_REFERENCE))
{
manifest->pub.referenceList = strLstNewSplitZ(varStr(jsonToVar(value)), ",");
loadData->referenceListFound = true;
}
else if (strEqZ(key, MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START))
manifest->pub.data.backupTimestampCopyStart = (time_t)varUInt64(jsonToVar(value));
else if (strEqZ(key, MANIFEST_KEY_BACKUP_TIMESTAMP_START))
@@ -2305,6 +2322,9 @@ manifestSaveCallback(void *const callbackData, const String *const sectionNext,
jsonFromVar(VARSTR(manifest->pub.data.backupLabelPrior)));
}
infoSaveValue(
infoSaveData, MANIFEST_SECTION_BACKUP, MANIFEST_KEY_BACKUP_REFERENCE,
jsonFromVar(VARSTR(strLstJoin(manifest->pub.referenceList, ","))));
infoSaveValue(
infoSaveData, MANIFEST_SECTION_BACKUP, MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START,
jsonFromVar(VARINT64(manifest->pub.data.backupTimestampCopyStart)));
@@ -3046,6 +3066,7 @@ manifestBackupLabelSet(Manifest *this, const String *backupLabel)
MEM_CONTEXT_BEGIN(this->pub.memContext)
{
this->pub.data.backupLabel = strDup(backupLabel);
strLstAdd(this->pub.referenceList, backupLabel);
}
MEM_CONTEXT_END();

8
src/info/manifest.h
View File

@@ -182,6 +182,7 @@ typedef struct ManifestPub
List *linkList; // List of links
List *pathList; // List of paths
List *targetList; // List of targets
StringList *referenceList; // List of file references
} ManifestPub;
// Get/set the cipher subpassphrase
@@ -204,6 +205,13 @@ manifestData(const Manifest *const this)
return &(THIS_PUB(Manifest)->data);
}
// Get reference list
FN_INLINE_ALWAYS const StringList *
manifestReferenceList(const Manifest *const this)
{
return THIS_PUB(Manifest)->referenceList;
}
// Set backup label
void manifestBackupLabelSet(Manifest *this, const String *backupLabel);

5
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
View File

@@ -575,6 +575,8 @@ sub backupCompare
my $oActualManifest = new pgBackRestTest::Env::Manifest(
$self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()});
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{'backup-reference'} =
$oActualManifest->get(MANIFEST_SECTION_BACKUP, 'backup-reference');
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} =
$oActualManifest->get(MANIFEST_SECTION_BACKUP, &MANIFEST_KEY_TIMESTAMP_START);
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP} =
@@ -2122,6 +2124,9 @@ sub restoreCompare
$oActualManifest->set(
MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef,
$oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL});
$oActualManifest->set(
MANIFEST_SECTION_BACKUP, 'backup-reference', undef,
$oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-reference'});
$oActualManifest->set(
MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef,
$oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START});

2
test/src/common/harnessLog.c
View File

@@ -353,7 +353,7 @@ hrnLogReplace(void)
if (logReplace->version)
{
unsigned int index = lstFindIdx((List *)logReplace->matchList, &match);
unsigned int index = strLstFindIdxP(logReplace->matchList, match);
if (index == LIST_NOT_FOUND)
{

1
test/src/module/command/backupTest.c
View File

@@ -2547,6 +2547,7 @@ testRun(void)
const String *resumeLabel = backupLabelCreate(
backupTypeDiff, manifestData(manifestPrior)->backupLabel, backupTimeStart);
manifestBackupLabelSet(manifestResume, resumeLabel);
strLstAddZ(manifestResume->pub.referenceList, "BOGUS");
// Reference in manifest
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), zNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION.gz", strZ(resumeLabel)));

5
test/src/module/command/restoreTest.c
View File

@@ -2427,6 +2427,11 @@ testRun(void)
manifest->pub.data.backupType = backupTypeIncr;
manifest->pub.data.backupTimestampCopyStart = 1482182861; // So file timestamps should be less than this
manifest->pub.referenceList = strLstNew();
strLstAddZ(manifest->pub.referenceList, TEST_LABEL_FULL);
strLstAddZ(manifest->pub.referenceList, TEST_LABEL_DIFF);
strLstAddZ(manifest->pub.referenceList, TEST_LABEL_INCR);
// Data directory
manifestTargetAdd(manifest, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = pgPath});
manifestPathAdd(

7
test/src/module/common/typeStringTest.c
View File

@@ -368,6 +368,13 @@ testRun(void)
TEST_RESULT_UINT(strLstSize(list), 9, "list size");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("strLstFindIdxP()");
TEST_RESULT_UINT(strLstFindIdxP(list, STRDEF("STR05")), 5, "find STR05");
TEST_RESULT_UINT(strLstFindIdxP(list, STRDEF("STR10")), LIST_NOT_FOUND, "find missing STR10");
TEST_ERROR(strLstFindIdxP(list, STRDEF("STR10"), .required = true), AssertError, "unable to find 'STR10' in string list");
// Read them back and check values
// -------------------------------------------------------------------------------------------------------------------------
for (unsigned int listIdx = 0; listIdx < strLstSize(list); listIdx++)

24
test/src/module/info/manifestTest.c
View File

@@ -40,6 +40,7 @@ testRun(void)
#define TEST_MANIFEST_HEADER \
"[backup]\n" \
"backup-label=null\n" \
"backup-reference=\"\"\n" \
"backup-timestamp-copy-start=0\n" \
"backup-timestamp-start=0\n" \
"backup-timestamp-stop=0\n" \
@@ -49,6 +50,7 @@ testRun(void)
"[backup]\n" \
"backup-bundle=true\n" \
"backup-label=null\n" \
"backup-reference=\"\"\n" \
"backup-timestamp-copy-start=0\n" \
"backup-timestamp-start=0\n" \
"backup-timestamp-stop=0\n" \
@@ -1002,7 +1004,9 @@ testRun(void)
#define TEST_MANIFEST_HEADER_PRE \
"[backup]\n" \
"backup-label=null\n" \
"backup-prior=\"20190101-010101F\"\n" \
"backup-prior=\"20190101-010101F\"\n"
#define TEST_MANIFEST_HEADER_MID \
"backup-timestamp-copy-start=0\n" \
"backup-timestamp-start=0\n" \
"backup-timestamp-stop=0\n" \
@@ -1085,6 +1089,7 @@ testRun(void)
{
manifestPrior = manifestNewInternal();
manifestPrior->pub.data.backupLabel = strNewZ("20190101-010101F");
strLstAdd(manifestPrior->pub.referenceList, manifestPrior->pub.data.backupLabel);
manifestFileAdd(
manifestPrior,
@@ -1112,6 +1117,8 @@ testRun(void)
strNewBuf(contentSave),
strNewBuf(harnessInfoChecksumZ(
TEST_MANIFEST_HEADER_PRE
"backup-reference=\"20190101-010101F\"\n"
TEST_MANIFEST_HEADER_MID
"option-delta=false\n"
TEST_MANIFEST_HEADER_POST
"\n"
@@ -1135,6 +1142,7 @@ testRun(void)
TEST_TITLE("delta enabled before validation");
manifest->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
strLstAddZ(manifestPrior->pub.referenceList, "20190101-010101F_20190202-010101D");
lstClear(manifest->pub.fileList);
manifestFileAdd(
manifest,
@@ -1186,6 +1194,8 @@ testRun(void)
strNewBuf(contentSave),
strNewBuf(harnessInfoChecksumZ(
TEST_MANIFEST_HEADER_PRE
"backup-reference=\"20190101-010101F,20190101-010101F_20190202-010101D\"\n"
TEST_MANIFEST_HEADER_MID
"option-delta=true\n"
TEST_MANIFEST_HEADER_POST
"\n"
@@ -1244,6 +1254,8 @@ testRun(void)
strNewBuf(contentSave),
strNewBuf(harnessInfoChecksumZ(
TEST_MANIFEST_HEADER_PRE
"backup-reference=\"20190101-010101F,20190101-010101F_20190202-010101D\"\n"
TEST_MANIFEST_HEADER_MID
"option-delta=true\n"
TEST_MANIFEST_HEADER_POST
"\n"
@@ -1298,6 +1310,8 @@ testRun(void)
strNewBuf(contentSave),
strNewBuf(harnessInfoChecksumZ(
TEST_MANIFEST_HEADER_PRE
"backup-reference=\"20190101-010101F,20190101-010101F_20190202-010101D\"\n"
TEST_MANIFEST_HEADER_MID
"option-delta=true\n"
TEST_MANIFEST_HEADER_POST
"\n"
@@ -1360,6 +1374,8 @@ testRun(void)
strNewBuf(contentSave),
strNewBuf(harnessInfoChecksumZ(
TEST_MANIFEST_HEADER_PRE
"backup-reference=\"20190101-010101F,20190101-010101F_20190202-010101D\"\n"
TEST_MANIFEST_HEADER_MID
"option-delta=true\n"
"option-hardlink=false\n"
"option-online=true\n"
@@ -1377,6 +1393,7 @@ testRun(void)
"check manifest");
#undef TEST_MANIFEST_HEADER_PRE
#undef TEST_MANIFEST_HEADER_MID
#undef TEST_MANIFEST_HEADER_POST
#undef TEST_MANIFEST_FILE_DEFAULT
#undef TEST_MANIFEST_PATH_DEFAULT
@@ -1392,6 +1409,7 @@ testRun(void)
(
"[backup]\n"
"backup-label=\"20190808-163540F\"\n"
"backup-reference=\"20190808-163540F\"\n"
"backup-timestamp-copy-start=1565282141\n"
"backup-timestamp-start=1565282140\n"
"backup-timestamp-stop=1565282142\n"
@@ -1419,7 +1437,8 @@ testRun(void)
"cipher-pass=\"somepass\"\n"
"\n"
"[target:file]\n"
"pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"size\":4,\"timestamp\":1565282114}\n"
"pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"reference\":\"20190808-163540F\""
",\"size\":4,\"timestamp\":1565282114}\n"
"\n"
"[target:file:default]\n"
"group=\"group1\"\n"
@@ -1477,6 +1496,7 @@ testRun(void)
"backup-lsn-start=\"285/89000028\"\n" \
"backup-lsn-stop=\"285/89001F88\"\n" \
"backup-prior=\"20190818-084502F\"\n" \
"backup-reference=\"20190818-084502F_20190819-084506D,20190818-084502F,20190818-084502F_20190820-084502D\"\n" \
"backup-timestamp-copy-start=1565282141\n" \
"backup-timestamp-start=1565282140\n" \
"backup-timestamp-stop=1565282142\n" \