1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-05 00:28:52 +02:00

Add manifest flags for file processing during backup.

The prior method was to check a combination of fields to determine if a file needed to be copied, delta'd, or resumed. This was complicated and ultimately imposed a limitation on the number of operations that could be performed.

Introduce copy, delta, and resume flags in the manifest to make it clearer which operations need to be performed and to reduce complex and duplicated logic.

This also allows zero-length bundled files to be completed during manifest build rather than later on during backup processing.
This commit is contained in:
David Steele
2022-10-05 13:14:15 -10:00
committed by GitHub
parent bb11539a37
commit c647bcb509
9 changed files with 241 additions and 153 deletions

View File

@ -51,10 +51,14 @@
<commit subject="Improve manifest file updates.">
<github-pull-request id="1888"/>
</commit>
<commit subject="Add manifest flags for file processing during backup.">
<github-pull-request id="1889"/>
</commit>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="stefan.fercot"/>
<release-item-reviewer id="john.morris"/>
</release-item-contributor-list>
<p>Block incremental backup.</p>

View File

@ -594,11 +594,16 @@ backupResumeClean(
removeReason = "zero size";
else
{
ASSERT(file.copy);
ASSERT(file.bundleId == 0);
file.sizeRepo = fileResume.sizeRepo;
memcpy(file.checksumSha1, fileResume.checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
file.checksumPage = fileResume.checksumPage;
file.checksumPageError = fileResume.checksumPageError;
file.checksumPageErrorList = fileResume.checksumPageErrorList;
file.resume = true;
file.delta = delta;
manifestFileUpdate(manifest, &file);
}
@ -1544,26 +1549,21 @@ backupProcessQueue(const BackupData *const backupData, Manifest *const manifest,
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
const ManifestFilePack *const filePack = manifestFilePackGet(manifest, fileIdx);
ManifestFile file = manifestFileUnpack(manifest, filePack);
const ManifestFile file = manifestFileUnpack(manifest, filePack);
// If bundling store zero-length files immediately in the manifest without copying them
if (jobData->bundle && file.size == 0)
// Only process files that need to be copied
if (!file.copy)
{
// If bundling log zero-length files as stored since they will never be copied
if (file.size == 0 && jobData->bundle)
{
LOG_DETAIL_FMT(
"store zero-length file %s", strZ(storagePathP(backupData->storagePrimary, manifestPathPg(file.name))));
memcpy(file.checksumSha1, HASH_TYPE_SHA1_ZERO, HASH_TYPE_SHA1_SIZE_HEX + 1);
file.reference = NULL;
manifestFileUpdate(manifest, &file);
}
continue;
}
// If the file is a reference it should only be backed up if delta and not zero size
if (file.reference != NULL && (!jobData->delta || file.size == 0))
continue;
// Is pg_control in the backup?
if (strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)))
pgControlFound = true;
@ -1722,18 +1722,19 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
pckWriteStrP(param, repoFile);
pckWriteU32P(param, jobData->compressType);
pckWriteI32P(param, jobData->compressLevel);
pckWriteBoolP(param, jobData->delta);
pckWriteU64P(param, jobData->cipherSubPass == NULL ? cipherTypeNone : cipherTypeAes256Cbc);
pckWriteStrP(param, jobData->cipherSubPass);
}
pckWriteStrP(param, manifestPathPg(file.name));
pckWriteBoolP(param, file.delta);
pckWriteBoolP(param, !strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)));
pckWriteU64P(param, file.size);
pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file.name));
pckWriteStrP(param, file.checksumSha1[0] != 0 ? STR(file.checksumSha1) : NULL);
pckWriteBoolP(param, file.checksumPage);
pckWriteStrP(param, file.name);
pckWriteBoolP(param, file.resume);
pckWriteBoolP(param, file.reference != NULL);
fileTotal++;

View File

@ -39,13 +39,12 @@ segmentNumber(const String *pgFile)
List *
backupFile(
const String *const repoFile, const CompressType repoFileCompressType, const int repoFileCompressLevel,
const bool delta, const CipherType cipherType, const String *const cipherPass, const List *const fileList)
const CipherType cipherType, const String *const cipherPass, const List *const fileList)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, repoFile); // Repo file
FUNCTION_LOG_PARAM(ENUM, repoFileCompressType); // Compress type for repo file
FUNCTION_LOG_PARAM(INT, repoFileCompressLevel); // Compression level for repo file
FUNCTION_LOG_PARAM(BOOL, delta); // Is the delta option on?
FUNCTION_LOG_PARAM(STRING_ID, cipherType); // Encryption type
FUNCTION_TEST_PARAM(STRING, cipherPass); // Password to access the repo file if encrypted
FUNCTION_LOG_PARAM(LIST, fileList); // List of files to backup
@ -71,24 +70,20 @@ backupFile(
const BackupFile *const file = lstGet(fileList, fileIdx);
ASSERT(file->pgFile != NULL);
ASSERT(file->manifestFile != NULL);
ASSERT((!file->pgFileDelta && !file->manifestFileResume) || file->pgFileChecksum != NULL);
BackupFileResult *const fileResult = lstAdd(
result, &(BackupFileResult){.manifestFile = file->manifestFile, .backupCopyResult = backupCopyResultCopy});
// If checksum is defined then the file needs to be checked. If delta option then check the DB and possibly the
// repo, else just check the repo.
if (file->pgFileChecksum != NULL)
{
// Does the file in pg match the checksum and size passed?
bool pgFileMatch = false;
// If delta, then check the DB checksum and possibly the repo. If the checksum does not match in either case
// then recopy.
if (delta)
// If delta then check the pg checksum
if (file->pgFileDelta)
{
// Generate checksum/size for the pg file. Only read as many bytes as passed in pgFileSize. If the file has
// grown since the manifest was built we don't need to consider the extra bytes since they will be replayed
// from WAL during recovery.
// grown since the manifest was built we don't need to consider the extra bytes since they will be replayed from
// WAL during recovery.
IoRead *read = storageReadIo(
storageNewReadP(
storagePg(), file->pgFile, .ignoreMissing = file->pgFileIgnoreMissing,
@ -126,21 +121,23 @@ backupFile(
fileResult->backupCopyResult = backupCopyResultSkip;
}
// If this is not a delta backup or it is and the file exists and the checksum from the DB matches, then also
// test the checksum of the file in the repo (unless it is in a prior backup) and if the checksum doesn't match,
// then there may be corruption in the repo, so recopy
if (!delta || !file->manifestFileHasReference)
// On resume check the manifest file
if (file->manifestFileResume)
{
// If this is a delta backup and the file is missing from the DB, then remove it from the repo
// (backupManifestUpdate will remove it from the manifest)
// Resumed files should never have a reference to a prior backup
ASSERT(!file->manifestFileHasReference);
// If the file is missing from pg, then remove it from the repo (backupJobResult() will remove it from the
// manifest)
if (fileResult->backupCopyResult == backupCopyResultSkip)
{
storageRemoveP(storageRepoWrite(), repoFile);
}
else if (!delta || pgFileMatch)
// Else if the pg file matches or is unknown because delta was not performed then check the repo file
else if (!file->pgFileDelta || pgFileMatch)
{
// Check the repo file in a try block because on error (e.g. missing or corrupt file that can't be
// decrypted or decompressed) we should recopy rather than ending the backup.
// Check the repo file in a try block because on error (e.g. missing or corrupt file that can't be decrypted
// or decompressed) we should recopy rather than ending the backup.
TRY_BEGIN()
{
// Generate checksum/size for the repo file
@ -191,7 +188,6 @@ backupFile(
}
}
}
}
MEM_CONTEXT_TEMP_END();
}

View File

@ -27,12 +27,14 @@ Functions
typedef struct BackupFile
{
const String *pgFile; // Pg file to backup
bool pgFileDelta; // Checksum pg file before copying
bool pgFileIgnoreMissing; // Ignore missing pg file
uint64_t pgFileSize; // Expected pg file size
bool pgFileCopyExactSize; // Copy only pg expected size
const String *pgFileChecksum; // Expected pg file checksum
bool pgFileChecksumPage; // Validate page checksums?
const String *manifestFile; // Repo file
bool manifestFileResume; // Checksum repo file before copying
bool manifestFileHasReference; // Reference to prior backup, if any
} BackupFile;
@ -48,7 +50,7 @@ typedef struct BackupFileResult
} BackupFileResult;
List *backupFile(
const String *repoFile, CompressType repoFileCompressType, int repoFileCompressLevel, bool delta, CipherType cipherType,
const String *repoFile, CompressType repoFileCompressType, int repoFileCompressLevel, CipherType cipherType,
const String *cipherPass, const List *fileList);
#endif

View File

@ -31,7 +31,6 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
const String *const repoFile = pckReadStrP(param);
const CompressType repoFileCompressType = (CompressType)pckReadU32P(param);
const int repoFileCompressLevel = pckReadI32P(param);
const bool delta = pckReadBoolP(param);
const CipherType cipherType = (CipherType)pckReadU64P(param);
const String *const cipherPass = pckReadStrP(param);
@ -41,12 +40,14 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
while (!pckReadNullP(param))
{
BackupFile file = {.pgFile = pckReadStrP(param)};
file.pgFileDelta = pckReadBoolP(param);
file.pgFileIgnoreMissing = pckReadBoolP(param);
file.pgFileSize = pckReadU64P(param);
file.pgFileCopyExactSize = pckReadBoolP(param);
file.pgFileChecksum = pckReadStrP(param);
file.pgFileChecksumPage = pckReadBoolP(param);
file.manifestFile = pckReadStrP(param);
file.manifestFileResume = pckReadBoolP(param);
file.manifestFileHasReference = pckReadBoolP(param);
lstAdd(fileList, &file);
@ -54,7 +55,7 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
// Backup file
const List *const result = backupFile(
repoFile, repoFileCompressType, repoFileCompressLevel, delta, cipherType, cipherPass, fileList);
repoFile, repoFileCompressType, repoFileCompressLevel, cipherType, cipherPass, fileList);
// Return result
PackWrite *const resultPack = protocolPackNew();

View File

@ -97,6 +97,9 @@ typedef enum
{
manifestFilePackFlagReference,
manifestFilePackFlagBundle,
manifestFilePackFlagCopy,
manifestFilePackFlagDelta,
manifestFilePackFlagResume,
manifestFilePackFlagChecksumPage,
manifestFilePackFlagChecksumPageError,
manifestFilePackFlagChecksumPageErrorList,
@ -125,6 +128,15 @@ manifestFilePack(const Manifest *const manifest, const ManifestFile *const file)
// Flags
uint64_t flag = 0;
if (file->copy)
flag |= 1 << manifestFilePackFlagCopy;
if (file->delta)
flag |= 1 << manifestFilePackFlagDelta;
if (file->resume)
flag |= 1 << manifestFilePackFlagResume;
if (file->checksumPage)
flag |= 1 << manifestFilePackFlagChecksumPage;
@ -251,6 +263,10 @@ manifestFileUnpack(const Manifest *const manifest, const ManifestFilePack *const
// Flags
const uint64_t flag = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX);
result.copy = (flag >> manifestFilePackFlagCopy) & 1;
result.delta = (flag >> manifestFilePackFlagDelta) & 1;
result.resume = (flag >> manifestFilePackFlagResume) & 1;
// Size
result.size = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX);
@ -259,7 +275,7 @@ manifestFileUnpack(const Manifest *const manifest, const ManifestFilePack *const
manifestPackBaseTime - (time_t)cvtInt64FromZigZag(cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX));
// Checksum page
result.checksumPage = flag & (1 << manifestFilePackFlagChecksumPage) ? true : false;
result.checksumPage = (flag >> manifestFilePackFlagChecksumPage) & 1;
// SHA1 checksum
memcpy(result.checksumSha1, (const uint8_t *)filePack + bufferPos, HASH_TYPE_SHA1_SIZE_HEX + 1);
@ -895,6 +911,7 @@ manifestBuildInfo(
ManifestFile file =
{
.name = manifestName,
.copy = true,
.mode = info->mode,
.user = info->user,
.group = info->group,
@ -903,6 +920,13 @@ manifestBuildInfo(
.timestamp = info->timeModified,
};
// When bundling zero-length files do not need to be copied
if (info->size == 0 && buildData->manifest->pub.data.bundle)
{
file.copy = false;
memcpy(file.checksumSha1, HASH_TYPE_SHA1_ZERO, HASH_TYPE_SHA1_SIZE_HEX + 1);
}
// Determine if this file should be page checksummed
if (dbPath && buildData->checksumPage)
{
@ -1484,7 +1508,7 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
{
const ManifestFile filePrior = manifestFileFind(manifestPrior, file.name);
if (file.size == filePrior.size && (delta || file.size == 0 || file.timestamp == filePrior.timestamp))
if (file.copy && file.size == filePrior.size && (delta || file.size == 0 || file.timestamp == filePrior.timestamp))
{
file.sizeRepo = filePrior.sizeRepo;
memcpy(file.checksumSha1, filePrior.checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
@ -1495,6 +1519,12 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
file.bundleId = filePrior.bundleId;
file.bundleOffset = filePrior.bundleOffset;
// Perform delta if the file size is not zero
file.delta = delta && file.size != 0;
// Copy if the file has changed or delta
file.copy = (file.size != 0 && file.timestamp != filePrior.timestamp) || file.delta;
manifestFileUpdate(this, &file);
}
}

View File

@ -99,6 +99,9 @@ File type
typedef struct ManifestFile
{
const String *name; // File name (must be first member in struct)
bool copy; // Should the file be copied (backup only)?
bool delta; // Verify checksum in PGDATA before copying (backup only)?
bool resume; // Is the file being resumed (backup only)?
bool checksumPage:1; // Does this file have page checksums?
bool checksumPageError:1; // Is there an error in the page checksum?
mode_t mode; // File mode

View File

@ -864,7 +864,7 @@ testRun(void)
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, false, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"pg file missing, ignoreMissing=true, no delta");
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy/repo size 0");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, "skip file");
@ -889,7 +889,7 @@ testRun(void)
lstAdd(fileList, &file);
TEST_ERROR(
backupFile(repoFile, compressTypeNone, 1, false, cipherTypeNone, NULL, fileList), FileMissingError,
backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), FileMissingError,
"unable to open missing file '" TEST_PATH "/pg/missing' for read");
// Create a pg file to backup
@ -925,7 +925,7 @@ testRun(void)
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, false, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"file checksummed with pageChecksum enabled");
TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 9, "repo=pgFile size");
@ -955,7 +955,7 @@ testRun(void)
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, false, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"backup file");
TEST_RESULT_UINT(result.copySize, 12, "copy size");
TEST_RESULT_UINT(result.repoSize, 12, "repo size");
@ -972,6 +972,7 @@ testRun(void)
file = (BackupFile)
{
.pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
@ -986,7 +987,7 @@ testRun(void)
// File exists in repo and db, pg checksum match, delta set, ignoreMissing false, hasReference - NOOP
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"file in db and repo, checksum equal, no ignoreMissing, no pageChecksum, delta, hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy size set");
TEST_RESULT_UINT(result.repoSize, 0, "repo size not set since already exists in repo");
@ -1003,6 +1004,7 @@ testRun(void)
file = (BackupFile)
{
.pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
@ -1017,7 +1019,7 @@ testRun(void)
// File exists in repo and db, pg checksum mismatch, delta set, ignoreMissing false, hasReference - COPY
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"file in db and repo, pg checksum not equal, no ignoreMissing, no pageChecksum, delta, hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
@ -1034,6 +1036,7 @@ testRun(void)
file = (BackupFile)
{
.pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false,
.pgFileSize = 9999999,
.pgFileCopyExactSize = true,
@ -1048,7 +1051,7 @@ testRun(void)
// File exists in repo and pg, pg checksum same, pg size passed is different, delta set, ignoreMissing false, hasReference
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"db & repo file, pg checksum same, pg size different, no ignoreMissing, no pageChecksum, delta, hasReference");
TEST_RESULT_UINT(result.copySize, 12, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 12, "repo=pgFile size");
@ -1065,12 +1068,14 @@ testRun(void)
file = (BackupFile)
{
.pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false,
.manifestFile = STRDEF(BOGUS_STR),
.manifestFileResume = true,
.manifestFileHasReference = false,
};
@ -1082,7 +1087,7 @@ testRun(void)
storageRepo(), STORAGE_REPO_BACKUP "/20190718-155825F", "testfile\n", .comment = "resumed file is missing in repo");
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"backup 9 bytes of pgfile to file to resume in repo");
TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
@ -1106,12 +1111,14 @@ testRun(void)
file = (BackupFile)
{
.pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false,
};
@ -1122,7 +1129,7 @@ testRun(void)
// Delta set, ignoreMissing false, no hasReference
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"db & repo file, pgFileMatch, repo checksum no match, no ignoreMissing, no pageChecksum, delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
@ -1139,12 +1146,14 @@ testRun(void)
file = (BackupFile)
{
.pgFile = missingFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = true,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false,
};
@ -1152,7 +1161,7 @@ testRun(void)
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"file in repo only, checksum in repo equal, ignoreMissing=true, no pageChecksum, delta, no hasReference");
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy=repo=0 size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, "skip file");
@ -1184,7 +1193,7 @@ testRun(void)
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeGz, 3, false, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeGz, 3, cipherTypeNone, NULL, fileList), 0),
"pg file exists, no checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 29, "repo compress size");
@ -1209,6 +1218,7 @@ testRun(void)
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false,
};
@ -1216,7 +1226,7 @@ testRun(void)
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeGz, 3, false, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeGz, 3, cipherTypeNone, NULL, fileList), 0),
"pg file & repo exists, match, checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 0, "repo size not calculated");
@ -1253,7 +1263,7 @@ testRun(void)
// No prior checksum, no compression, no pageChecksum, no delta, no hasReference
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, false, cipherTypeNone, NULL, fileList), 0),
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"zero-sized pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy=repo=pgFile size 0");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
@ -1305,7 +1315,7 @@ testRun(void)
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(
backupFile(repoFile, compressTypeNone, 1, false, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
backupFile(repoFile, compressTypeNone, 1, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
"pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy size set");
TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
@ -1324,12 +1334,14 @@ testRun(void)
file = (BackupFile)
{
.pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false,
.pgFileSize = 8,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false,
};
@ -1339,7 +1351,7 @@ testRun(void)
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(
backupFile(repoFile, compressTypeNone, 1, true, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
backupFile(repoFile, compressTypeNone, 1, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
"pg and repo file exists, pgFileMatch false, no ignoreMissing, no pageChecksum, delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 8, "copy size set");
TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
@ -1364,6 +1376,7 @@ testRun(void)
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false,
};
@ -1372,7 +1385,7 @@ testRun(void)
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(
backupFile(repoFile, compressTypeNone, 0, false, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
backupFile(repoFile, compressTypeNone, 0, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
"pg and repo file exists, checksum mismatch, no ignoreMissing, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy size set");
TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
@ -1397,6 +1410,7 @@ testRun(void)
.pgFileChecksum = STRDEF("1234567890123456789012345678901234567890"),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false,
};
@ -1405,7 +1419,7 @@ testRun(void)
TEST_ASSIGN(
result,
*(BackupFileResult *)lstGet(
backupFile(repoFile, compressTypeNone, 0, false, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
backupFile(repoFile, compressTypeNone, 0, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
"backup file");
TEST_RESULT_UINT(result.copySize, 9, "copy size set");

View File

@ -45,6 +45,15 @@ testRun(void)
"backup-timestamp-stop=0\n" \
"backup-type=\"full\"\n"
#define TEST_MANIFEST_HEADER_BUNDLE \
"[backup]\n" \
"backup-bundle=true\n" \
"backup-label=null\n" \
"backup-timestamp-copy-start=0\n" \
"backup-timestamp-start=0\n" \
"backup-timestamp-stop=0\n" \
"backup-type=\"full\"\n"
#define TEST_MANIFEST_DB_90 \
"\n" \
"[backup:db]\n" \
@ -801,7 +810,7 @@ testRun(void)
// pg_wal not ignored
TEST_ASSIGN(
manifest,
manifestNewBuild(storagePg, PG_VERSION_13, hrnPgCatalogVersion(PG_VERSION_13), false, false, false, NULL, NULL),
manifestNewBuild(storagePg, PG_VERSION_13, hrnPgCatalogVersion(PG_VERSION_13), false, false, true, NULL, NULL),
"build manifest");
contentSave = bufNew(0);
@ -809,7 +818,7 @@ testRun(void)
TEST_RESULT_STR(
strNewBuf(contentSave),
strNewBuf(harnessInfoChecksumZ(
TEST_MANIFEST_HEADER
TEST_MANIFEST_HEADER_BUNDLE
TEST_MANIFEST_DB_13
TEST_MANIFEST_OPTION_ALL
"\n"
@ -1050,23 +1059,23 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/BOGUS"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
.name = STRDEF(MANIFEST_TARGET_PGDATA "/BOGUS"), .copy = true, .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .copy = true, .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182861,
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .copy = true, .size = 55, .sizeRepo = 55, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .copy = true, .size = 4, .sizeRepo = 4,
.timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
}
OBJ_NEW_END();
@ -1130,13 +1139,26 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .copy = true, .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
// Zero-length file without the copy flag which will appear to come from a bundled backup
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-bundle"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
// Zero-length file with the copy flag which will appear to come from a non-bundled backup (so will get a reference)
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-normal"), .copy = true, .size = 0, .sizeRepo = 0,
.timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"),
.checksumSha1 = HASH_TYPE_SHA1_ZERO});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .copy = true, .size = 4, .sizeRepo = 4,
.timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifestPrior,
@ -1144,6 +1166,17 @@ testRun(void)
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-bundle"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-normal"), .size = 0, .sizeRepo = 0,
.timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"),
.checksumSha1 = HASH_TYPE_SHA1_ZERO});
TEST_RESULT_VOID(manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, NULL), "incremental manifest");
@ -1160,6 +1193,8 @@ testRun(void)
"pg_data={\"path\":\"/pg\",\"type\":\"path\"}\n"
"\n"
"[target:file]\n"
"pg_data/FILE0-bundle={\"size\":0,\"timestamp\":1482182860}\n"
"pg_data/FILE0-normal={\"reference\":\"20190101-010101F\",\"size\":0,\"timestamp\":1482182860}\n"
"pg_data/FILE1={\"checksum\":\"aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd\","
"\"reference\":\"20190101-010101F_20190202-010101D\",\"size\":4,\"timestamp\":1482182860}\n"
"pg_data/PG_VERSION={\"checksum\":\"aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd\",\"reference\":\"20190101-010101F\","
@ -1181,7 +1216,7 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182859,
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .copy = true, .size = 4, .sizeRepo = 4, .timestamp = 1482182859,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
// Clear prior manifest and add a single file with later timestamp and checksum error
@ -1192,7 +1227,7 @@ testRun(void)
manifestFileAdd(
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .copy = true, .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", .checksumPage = true, .checksumPageError = true,
.checksumPageErrorList = jsonFromVar(varNewVarLst(checksumPageErrorList))});
@ -1234,18 +1269,18 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .copy = true, .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .copy = true, .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .copy = true, .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
@ -1304,7 +1339,7 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .copy = true, .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifest->pub.data.backupOptionOnline = BOOL_TRUE_VAR;
@ -1609,6 +1644,7 @@ testRun(void)
ManifestFile file = manifestFileFind(manifest, STRDEF("pg_data/postgresql.conf"));
file.checksumSha1[0] = '\0';
file.sizeRepo = 0;
file.resume = true;
manifestFileUpdate(manifest, &file);
file = manifestFileFind(manifest, STRDEF("pg_data/base/32768/33000.32767"));
@ -1629,6 +1665,7 @@ testRun(void)
// Undo changes made to files
file = manifestFileFind(manifest, STRDEF("pg_data/postgresql.conf"));
TEST_RESULT_BOOL(file.resume, true, "resume is set");
memcpy(file.checksumSha1, "184473f470864e067ee3a22e64b47b0a1c356f29", HASH_TYPE_SHA1_SIZE_HEX + 1);
file.sizeRepo = 4457;
manifestFileUpdate(manifest, &file);