db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-07 00:35:37 +02:00
Code Issues Releases Activity

Add manifest flags for file processing during backup.

Browse Source 
The prior method was to check a combination of fields to determine if a file needed to be copied, delta'd, or resumed. This was complicated and ultimately imposed a limitation on the number of operations that could be performed.

Introduce copy, delta, and resume flags in the manifest to make it clearer which operations need to be performed and to reduce complex and duplicated logic.

This also allows zero-length bundled files to be completed during manifest build rather than later on during backup processing.
This commit is contained in:
David Steele
2022-10-05 13:14:15 -10:00
committed by GitHub
parent bb11539a37
commit c647bcb509
9 changed files with 241 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
doc/xml/release.xml
View File

@ -51,10 +51,14 @@
<commit subject="Improve manifest file updates."> <commit subject="Improve manifest file updates.">
<github-pull-request id="1888"/> <github-pull-request id="1888"/>
</commit> </commit>
<commit subject="Add manifest flags for file processing during backup.">
<github-pull-request id="1889"/>
</commit>
<release-item-contributor-list> <release-item-contributor-list>
<release-item-contributor id="david.steele"/> <release-item-contributor id="david.steele"/>
<release-item-reviewer id="stefan.fercot"/> <release-item-reviewer id="stefan.fercot"/>
<release-item-reviewer id="john.morris"/>
</release-item-contributor-list> </release-item-contributor-list>
<p>Block incremental backup.</p> <p>Block incremental backup.</p>

27
src/command/backup/backup.c
View File

@ -594,11 +594,16 @@ backupResumeClean(
removeReason = "zero size"; removeReason = "zero size";
else else
{ {
ASSERT(file.copy);
ASSERT(file.bundleId == 0);
file.sizeRepo = fileResume.sizeRepo; file.sizeRepo = fileResume.sizeRepo;
memcpy(file.checksumSha1, fileResume.checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1); memcpy(file.checksumSha1, fileResume.checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
file.checksumPage = fileResume.checksumPage; file.checksumPage = fileResume.checksumPage;
file.checksumPageError = fileResume.checksumPageError; file.checksumPageError = fileResume.checksumPageError;
file.checksumPageErrorList = fileResume.checksumPageErrorList; file.checksumPageErrorList = fileResume.checksumPageErrorList;
file.resume = true;
file.delta = delta;
manifestFileUpdate(manifest, &file); manifestFileUpdate(manifest, &file);
} }
@ -1544,26 +1549,21 @@ backupProcessQueue(const BackupData *const backupData, Manifest *const manifest,
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{ {
const ManifestFilePack *const filePack = manifestFilePackGet(manifest, fileIdx); const ManifestFilePack *const filePack = manifestFilePackGet(manifest, fileIdx);
ManifestFile file = manifestFileUnpack(manifest, filePack); const ManifestFile file = manifestFileUnpack(manifest, filePack);
// If bundling store zero-length files immediately in the manifest without copying them // Only process files that need to be copied
if (jobData->bundle && file.size == 0) if (!file.copy)
{
// If bundling log zero-length files as stored since they will never be copied
if (file.size == 0 && jobData->bundle)
{ {
LOG_DETAIL_FMT( LOG_DETAIL_FMT(
"store zero-length file %s", strZ(storagePathP(backupData->storagePrimary, manifestPathPg(file.name)))); "store zero-length file %s", strZ(storagePathP(backupData->storagePrimary, manifestPathPg(file.name))));
}
memcpy(file.checksumSha1, HASH_TYPE_SHA1_ZERO, HASH_TYPE_SHA1_SIZE_HEX + 1);
file.reference = NULL;
manifestFileUpdate(manifest, &file);
continue; continue;
} }
// If the file is a reference it should only be backed up if delta and not zero size
if (file.reference != NULL && (!jobData->delta || file.size == 0))
continue;
// Is pg_control in the backup? // Is pg_control in the backup?
if (strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))) if (strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)))
pgControlFound = true; pgControlFound = true;
@ -1722,18 +1722,19 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
pckWriteStrP(param, repoFile); pckWriteStrP(param, repoFile);
pckWriteU32P(param, jobData->compressType); pckWriteU32P(param, jobData->compressType);
pckWriteI32P(param, jobData->compressLevel); pckWriteI32P(param, jobData->compressLevel);
pckWriteBoolP(param, jobData->delta);
pckWriteU64P(param, jobData->cipherSubPass == NULL ? cipherTypeNone : cipherTypeAes256Cbc); pckWriteU64P(param, jobData->cipherSubPass == NULL ? cipherTypeNone : cipherTypeAes256Cbc);
pckWriteStrP(param, jobData->cipherSubPass); pckWriteStrP(param, jobData->cipherSubPass);
} }
pckWriteStrP(param, manifestPathPg(file.name)); pckWriteStrP(param, manifestPathPg(file.name));
pckWriteBoolP(param, file.delta);
pckWriteBoolP(param, !strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))); pckWriteBoolP(param, !strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)));
pckWriteU64P(param, file.size); pckWriteU64P(param, file.size);
pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file.name)); pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file.name));
pckWriteStrP(param, file.checksumSha1[0] != 0 ? STR(file.checksumSha1) : NULL); pckWriteStrP(param, file.checksumSha1[0] != 0 ? STR(file.checksumSha1) : NULL);
pckWriteBoolP(param, file.checksumPage); pckWriteBoolP(param, file.checksumPage);
pckWriteStrP(param, file.name); pckWriteStrP(param, file.name);
pckWriteBoolP(param, file.resume);
pckWriteBoolP(param, file.reference != NULL); pckWriteBoolP(param, file.reference != NULL);
fileTotal++; fileTotal++;

38
src/command/backup/file.c
View File

@ -39,13 +39,12 @@ segmentNumber(const String *pgFile)
List * List *
backupFile( backupFile(
const String *const repoFile, const CompressType repoFileCompressType, const int repoFileCompressLevel, const String *const repoFile, const CompressType repoFileCompressType, const int repoFileCompressLevel,
const bool delta, const CipherType cipherType, const String *const cipherPass, const List *const fileList) const CipherType cipherType, const String *const cipherPass, const List *const fileList)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, repoFile); // Repo file FUNCTION_LOG_PARAM(STRING, repoFile); // Repo file
FUNCTION_LOG_PARAM(ENUM, repoFileCompressType); // Compress type for repo file FUNCTION_LOG_PARAM(ENUM, repoFileCompressType); // Compress type for repo file
FUNCTION_LOG_PARAM(INT, repoFileCompressLevel); // Compression level for repo file FUNCTION_LOG_PARAM(INT, repoFileCompressLevel); // Compression level for repo file
FUNCTION_LOG_PARAM(BOOL, delta); // Is the delta option on?
FUNCTION_LOG_PARAM(STRING_ID, cipherType); // Encryption type FUNCTION_LOG_PARAM(STRING_ID, cipherType); // Encryption type
FUNCTION_TEST_PARAM(STRING, cipherPass); // Password to access the repo file if encrypted FUNCTION_TEST_PARAM(STRING, cipherPass); // Password to access the repo file if encrypted
FUNCTION_LOG_PARAM(LIST, fileList); // List of files to backup FUNCTION_LOG_PARAM(LIST, fileList); // List of files to backup
@ -71,24 +70,20 @@ backupFile(
const BackupFile *const file = lstGet(fileList, fileIdx); const BackupFile *const file = lstGet(fileList, fileIdx);
ASSERT(file->pgFile != NULL); ASSERT(file->pgFile != NULL);
ASSERT(file->manifestFile != NULL); ASSERT(file->manifestFile != NULL);
ASSERT((!file->pgFileDelta && !file->manifestFileResume) || file->pgFileChecksum != NULL);
BackupFileResult *const fileResult = lstAdd( BackupFileResult *const fileResult = lstAdd(
result, &(BackupFileResult){.manifestFile = file->manifestFile, .backupCopyResult = backupCopyResultCopy}); result, &(BackupFileResult){.manifestFile = file->manifestFile, .backupCopyResult = backupCopyResultCopy});
// If checksum is defined then the file needs to be checked. If delta option then check the DB and possibly the
// repo, else just check the repo.
if (file->pgFileChecksum != NULL)
{
// Does the file in pg match the checksum and size passed? // Does the file in pg match the checksum and size passed?
bool pgFileMatch = false; bool pgFileMatch = false;
// If delta, then check the DB checksum and possibly the repo. If the checksum does not match in either case // If delta then check the pg checksum
// then recopy. if (file->pgFileDelta)
if (delta)
{ {
// Generate checksum/size for the pg file. Only read as many bytes as passed in pgFileSize. If the file has // Generate checksum/size for the pg file. Only read as many bytes as passed in pgFileSize. If the file has
// grown since the manifest was built we don't need to consider the extra bytes since they will be replayed // grown since the manifest was built we don't need to consider the extra bytes since they will be replayed from
// from WAL during recovery. // WAL during recovery.
IoRead *read = storageReadIo( IoRead *read = storageReadIo(
storageNewReadP( storageNewReadP(
storagePg(), file->pgFile, .ignoreMissing = file->pgFileIgnoreMissing, storagePg(), file->pgFile, .ignoreMissing = file->pgFileIgnoreMissing,
@ -126,21 +121,23 @@ backupFile(
fileResult->backupCopyResult = backupCopyResultSkip; fileResult->backupCopyResult = backupCopyResultSkip;
} }
// If this is not a delta backup or it is and the file exists and the checksum from the DB matches, then also // On resume check the manifest file
// test the checksum of the file in the repo (unless it is in a prior backup) and if the checksum doesn't match, if (file->manifestFileResume)
// then there may be corruption in the repo, so recopy
if (!delta || !file->manifestFileHasReference)
{ {
// If this is a delta backup and the file is missing from the DB, then remove it from the repo // Resumed files should never have a reference to a prior backup
// (backupManifestUpdate will remove it from the manifest) ASSERT(!file->manifestFileHasReference);
// If the file is missing from pg, then remove it from the repo (backupJobResult() will remove it from the
// manifest)
if (fileResult->backupCopyResult == backupCopyResultSkip) if (fileResult->backupCopyResult == backupCopyResultSkip)
{ {
storageRemoveP(storageRepoWrite(), repoFile); storageRemoveP(storageRepoWrite(), repoFile);
} }
else if (!delta || pgFileMatch) // Else if the pg file matches or is unknown because delta was not performed then check the repo file
else if (!file->pgFileDelta || pgFileMatch)
{ {
// Check the repo file in a try block because on error (e.g. missing or corrupt file that can't be // Check the repo file in a try block because on error (e.g. missing or corrupt file that can't be decrypted
// decrypted or decompressed) we should recopy rather than ending the backup. // or decompressed) we should recopy rather than ending the backup.
TRY_BEGIN() TRY_BEGIN()
{ {
// Generate checksum/size for the repo file // Generate checksum/size for the repo file
@ -191,7 +188,6 @@ backupFile(
} }
} }
} }
}
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
} }

4
src/command/backup/file.h
View File

@ -27,12 +27,14 @@ Functions
typedef struct BackupFile typedef struct BackupFile
{ {
const String *pgFile; // Pg file to backup const String *pgFile; // Pg file to backup
bool pgFileDelta; // Checksum pg file before copying
bool pgFileIgnoreMissing; // Ignore missing pg file bool pgFileIgnoreMissing; // Ignore missing pg file
uint64_t pgFileSize; // Expected pg file size uint64_t pgFileSize; // Expected pg file size
bool pgFileCopyExactSize; // Copy only pg expected size bool pgFileCopyExactSize; // Copy only pg expected size
const String *pgFileChecksum; // Expected pg file checksum const String *pgFileChecksum; // Expected pg file checksum
bool pgFileChecksumPage; // Validate page checksums? bool pgFileChecksumPage; // Validate page checksums?
const String *manifestFile; // Repo file const String *manifestFile; // Repo file
bool manifestFileResume; // Checksum repo file before copying
bool manifestFileHasReference; // Reference to prior backup, if any bool manifestFileHasReference; // Reference to prior backup, if any
} BackupFile; } BackupFile;
@ -48,7 +50,7 @@ typedef struct BackupFileResult
} BackupFileResult; } BackupFileResult;
List *backupFile( List *backupFile(
const String *repoFile, CompressType repoFileCompressType, int repoFileCompressLevel, bool delta, CipherType cipherType, const String *repoFile, CompressType repoFileCompressType, int repoFileCompressLevel, CipherType cipherType,
const String *cipherPass, const List *fileList); const String *cipherPass, const List *fileList);
#endif #endif

5
src/command/backup/protocol.c
View File

@ -31,7 +31,6 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
const String *const repoFile = pckReadStrP(param); const String *const repoFile = pckReadStrP(param);
const CompressType repoFileCompressType = (CompressType)pckReadU32P(param); const CompressType repoFileCompressType = (CompressType)pckReadU32P(param);
const int repoFileCompressLevel = pckReadI32P(param); const int repoFileCompressLevel = pckReadI32P(param);
const bool delta = pckReadBoolP(param);
const CipherType cipherType = (CipherType)pckReadU64P(param); const CipherType cipherType = (CipherType)pckReadU64P(param);
const String *const cipherPass = pckReadStrP(param); const String *const cipherPass = pckReadStrP(param);
@ -41,12 +40,14 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
while (!pckReadNullP(param)) while (!pckReadNullP(param))
{ {
BackupFile file = {.pgFile = pckReadStrP(param)}; BackupFile file = {.pgFile = pckReadStrP(param)};
file.pgFileDelta = pckReadBoolP(param);
file.pgFileIgnoreMissing = pckReadBoolP(param); file.pgFileIgnoreMissing = pckReadBoolP(param);
file.pgFileSize = pckReadU64P(param); file.pgFileSize = pckReadU64P(param);
file.pgFileCopyExactSize = pckReadBoolP(param); file.pgFileCopyExactSize = pckReadBoolP(param);
file.pgFileChecksum = pckReadStrP(param); file.pgFileChecksum = pckReadStrP(param);
file.pgFileChecksumPage = pckReadBoolP(param); file.pgFileChecksumPage = pckReadBoolP(param);
file.manifestFile = pckReadStrP(param); file.manifestFile = pckReadStrP(param);
file.manifestFileResume = pckReadBoolP(param);
file.manifestFileHasReference = pckReadBoolP(param); file.manifestFileHasReference = pckReadBoolP(param);
lstAdd(fileList, &file); lstAdd(fileList, &file);
@ -54,7 +55,7 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
// Backup file // Backup file
const List *const result = backupFile( const List *const result = backupFile(
repoFile, repoFileCompressType, repoFileCompressLevel, delta, cipherType, cipherPass, fileList); repoFile, repoFileCompressType, repoFileCompressLevel, cipherType, cipherPass, fileList);
// Return result // Return result
PackWrite *const resultPack = protocolPackNew(); PackWrite *const resultPack = protocolPackNew();

34
src/info/manifest.c
View File

@ -97,6 +97,9 @@ typedef enum
{ {
manifestFilePackFlagReference, manifestFilePackFlagReference,
manifestFilePackFlagBundle, manifestFilePackFlagBundle,
manifestFilePackFlagCopy,
manifestFilePackFlagDelta,
manifestFilePackFlagResume,
manifestFilePackFlagChecksumPage, manifestFilePackFlagChecksumPage,
manifestFilePackFlagChecksumPageError, manifestFilePackFlagChecksumPageError,
manifestFilePackFlagChecksumPageErrorList, manifestFilePackFlagChecksumPageErrorList,
@ -125,6 +128,15 @@ manifestFilePack(const Manifest *const manifest, const ManifestFile *const file)
// Flags // Flags
uint64_t flag = 0; uint64_t flag = 0;
if (file->copy)
flag |= 1 << manifestFilePackFlagCopy;
if (file->delta)
flag |= 1 << manifestFilePackFlagDelta;
if (file->resume)
flag |= 1 << manifestFilePackFlagResume;
if (file->checksumPage) if (file->checksumPage)
flag |= 1 << manifestFilePackFlagChecksumPage; flag |= 1 << manifestFilePackFlagChecksumPage;
@ -251,6 +263,10 @@ manifestFileUnpack(const Manifest *const manifest, const ManifestFilePack *const
// Flags // Flags
const uint64_t flag = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX); const uint64_t flag = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX);
result.copy = (flag >> manifestFilePackFlagCopy) & 1;
result.delta = (flag >> manifestFilePackFlagDelta) & 1;
result.resume = (flag >> manifestFilePackFlagResume) & 1;
// Size // Size
result.size = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX); result.size = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX);
@ -259,7 +275,7 @@ manifestFileUnpack(const Manifest *const manifest, const ManifestFilePack *const
manifestPackBaseTime - (time_t)cvtInt64FromZigZag(cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX)); manifestPackBaseTime - (time_t)cvtInt64FromZigZag(cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX));
// Checksum page // Checksum page
result.checksumPage = flag & (1 << manifestFilePackFlagChecksumPage) ? true : false; result.checksumPage = (flag >> manifestFilePackFlagChecksumPage) & 1;
// SHA1 checksum // SHA1 checksum
memcpy(result.checksumSha1, (const uint8_t *)filePack + bufferPos, HASH_TYPE_SHA1_SIZE_HEX + 1); memcpy(result.checksumSha1, (const uint8_t *)filePack + bufferPos, HASH_TYPE_SHA1_SIZE_HEX + 1);
@ -895,6 +911,7 @@ manifestBuildInfo(
ManifestFile file = ManifestFile file =
{ {
.name = manifestName, .name = manifestName,
.copy = true,
.mode = info->mode, .mode = info->mode,
.user = info->user, .user = info->user,
.group = info->group, .group = info->group,
@ -903,6 +920,13 @@ manifestBuildInfo(
.timestamp = info->timeModified, .timestamp = info->timeModified,
}; };
// When bundling zero-length files do not need to be copied
if (info->size == 0 && buildData->manifest->pub.data.bundle)
{
file.copy = false;
memcpy(file.checksumSha1, HASH_TYPE_SHA1_ZERO, HASH_TYPE_SHA1_SIZE_HEX + 1);
}
// Determine if this file should be page checksummed // Determine if this file should be page checksummed
if (dbPath && buildData->checksumPage) if (dbPath && buildData->checksumPage)
{ {
@ -1484,7 +1508,7 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
{ {
const ManifestFile filePrior = manifestFileFind(manifestPrior, file.name); const ManifestFile filePrior = manifestFileFind(manifestPrior, file.name);
if (file.size == filePrior.size && (delta || file.size == 0 || file.timestamp == filePrior.timestamp)) if (file.copy && file.size == filePrior.size && (delta || file.size == 0 || file.timestamp == filePrior.timestamp))
{ {
file.sizeRepo = filePrior.sizeRepo; file.sizeRepo = filePrior.sizeRepo;
memcpy(file.checksumSha1, filePrior.checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1); memcpy(file.checksumSha1, filePrior.checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
@ -1495,6 +1519,12 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
file.bundleId = filePrior.bundleId; file.bundleId = filePrior.bundleId;
file.bundleOffset = filePrior.bundleOffset; file.bundleOffset = filePrior.bundleOffset;
// Perform delta if the file size is not zero
file.delta = delta && file.size != 0;
// Copy if the file has changed or delta
file.copy = (file.size != 0 && file.timestamp != filePrior.timestamp) || file.delta;
manifestFileUpdate(this, &file); manifestFileUpdate(this, &file);
} }
} }

3
src/info/manifest.h
View File

@ -99,6 +99,9 @@ File type
typedef struct ManifestFile typedef struct ManifestFile
{ {
const String *name; // File name (must be first member in struct) const String *name; // File name (must be first member in struct)
bool copy; // Should the file be copied (backup only)?
bool delta; // Verify checksum in PGDATA before copying (backup only)?
bool resume; // Is the file being resumed (backup only)?
bool checksumPage:1; // Does this file have page checksums? bool checksumPage:1; // Does this file have page checksums?
bool checksumPageError:1; // Is there an error in the page checksum? bool checksumPageError:1; // Is there an error in the page checksum?
mode_t mode; // File mode mode_t mode; // File mode

48
test/src/module/command/backupTest.c
View File

@ -864,7 +864,7 @@ testRun(void)
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, false, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"pg file missing, ignoreMissing=true, no delta"); "pg file missing, ignoreMissing=true, no delta");
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy/repo size 0"); TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy/repo size 0");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, "skip file"); TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, "skip file");
@ -889,7 +889,7 @@ testRun(void)
lstAdd(fileList, &file); lstAdd(fileList, &file);
TEST_ERROR( TEST_ERROR(
backupFile(repoFile, compressTypeNone, 1, false, cipherTypeNone, NULL, fileList), FileMissingError, backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), FileMissingError,
"unable to open missing file '" TEST_PATH "/pg/missing' for read"); "unable to open missing file '" TEST_PATH "/pg/missing' for read");
// Create a pg file to backup // Create a pg file to backup
@ -925,7 +925,7 @@ testRun(void)
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, false, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"file checksummed with pageChecksum enabled"); "file checksummed with pageChecksum enabled");
TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size"); TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 9, "repo=pgFile size"); TEST_RESULT_UINT(result.repoSize, 9, "repo=pgFile size");
@ -955,7 +955,7 @@ testRun(void)
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, false, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"backup file"); "backup file");
TEST_RESULT_UINT(result.copySize, 12, "copy size"); TEST_RESULT_UINT(result.copySize, 12, "copy size");
TEST_RESULT_UINT(result.repoSize, 12, "repo size"); TEST_RESULT_UINT(result.repoSize, 12, "repo size");
@ -972,6 +972,7 @@ testRun(void)
file = (BackupFile) file = (BackupFile)
{ {
.pgFile = pgFile, .pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false, .pgFileIgnoreMissing = false,
.pgFileSize = 9, .pgFileSize = 9,
.pgFileCopyExactSize = true, .pgFileCopyExactSize = true,
@ -986,7 +987,7 @@ testRun(void)
// File exists in repo and db, pg checksum match, delta set, ignoreMissing false, hasReference - NOOP // File exists in repo and db, pg checksum match, delta set, ignoreMissing false, hasReference - NOOP
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"file in db and repo, checksum equal, no ignoreMissing, no pageChecksum, delta, hasReference"); "file in db and repo, checksum equal, no ignoreMissing, no pageChecksum, delta, hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy size set"); TEST_RESULT_UINT(result.copySize, 9, "copy size set");
TEST_RESULT_UINT(result.repoSize, 0, "repo size not set since already exists in repo"); TEST_RESULT_UINT(result.repoSize, 0, "repo size not set since already exists in repo");
@ -1003,6 +1004,7 @@ testRun(void)
file = (BackupFile) file = (BackupFile)
{ {
.pgFile = pgFile, .pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false, .pgFileIgnoreMissing = false,
.pgFileSize = 9, .pgFileSize = 9,
.pgFileCopyExactSize = true, .pgFileCopyExactSize = true,
@ -1017,7 +1019,7 @@ testRun(void)
// File exists in repo and db, pg checksum mismatch, delta set, ignoreMissing false, hasReference - COPY // File exists in repo and db, pg checksum mismatch, delta set, ignoreMissing false, hasReference - COPY
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"file in db and repo, pg checksum not equal, no ignoreMissing, no pageChecksum, delta, hasReference"); "file in db and repo, pg checksum not equal, no ignoreMissing, no pageChecksum, delta, hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes"); TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size"); TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
@ -1034,6 +1036,7 @@ testRun(void)
file = (BackupFile) file = (BackupFile)
{ {
.pgFile = pgFile, .pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false, .pgFileIgnoreMissing = false,
.pgFileSize = 9999999, .pgFileSize = 9999999,
.pgFileCopyExactSize = true, .pgFileCopyExactSize = true,
@ -1048,7 +1051,7 @@ testRun(void)
// File exists in repo and pg, pg checksum same, pg size passed is different, delta set, ignoreMissing false, hasReference // File exists in repo and pg, pg checksum same, pg size passed is different, delta set, ignoreMissing false, hasReference
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"db & repo file, pg checksum same, pg size different, no ignoreMissing, no pageChecksum, delta, hasReference"); "db & repo file, pg checksum same, pg size different, no ignoreMissing, no pageChecksum, delta, hasReference");
TEST_RESULT_UINT(result.copySize, 12, "copy=pgFile size"); TEST_RESULT_UINT(result.copySize, 12, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 12, "repo=pgFile size"); TEST_RESULT_UINT(result.repoSize, 12, "repo=pgFile size");
@ -1065,12 +1068,14 @@ testRun(void)
file = (BackupFile) file = (BackupFile)
{ {
.pgFile = pgFile, .pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false, .pgFileIgnoreMissing = false,
.pgFileSize = 9, .pgFileSize = 9,
.pgFileCopyExactSize = true, .pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), .pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false, .pgFileChecksumPage = false,
.manifestFile = STRDEF(BOGUS_STR), .manifestFile = STRDEF(BOGUS_STR),
.manifestFileResume = true,
.manifestFileHasReference = false, .manifestFileHasReference = false,
}; };
@ -1082,7 +1087,7 @@ testRun(void)
storageRepo(), STORAGE_REPO_BACKUP "/20190718-155825F", "testfile\n", .comment = "resumed file is missing in repo"); storageRepo(), STORAGE_REPO_BACKUP "/20190718-155825F", "testfile\n", .comment = "resumed file is missing in repo");
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"backup 9 bytes of pgfile to file to resume in repo"); "backup 9 bytes of pgfile to file to resume in repo");
TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes"); TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size"); TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
@ -1106,12 +1111,14 @@ testRun(void)
file = (BackupFile) file = (BackupFile)
{ {
.pgFile = pgFile, .pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false, .pgFileIgnoreMissing = false,
.pgFileSize = 9, .pgFileSize = 9,
.pgFileCopyExactSize = true, .pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), .pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false, .pgFileChecksumPage = false,
.manifestFile = pgFile, .manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false, .manifestFileHasReference = false,
}; };
@ -1122,7 +1129,7 @@ testRun(void)
// Delta set, ignoreMissing false, no hasReference // Delta set, ignoreMissing false, no hasReference
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"db & repo file, pgFileMatch, repo checksum no match, no ignoreMissing, no pageChecksum, delta, no hasReference"); "db & repo file, pgFileMatch, repo checksum no match, no ignoreMissing, no pageChecksum, delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes"); TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size"); TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
@ -1139,12 +1146,14 @@ testRun(void)
file = (BackupFile) file = (BackupFile)
{ {
.pgFile = missingFile, .pgFile = missingFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = true, .pgFileIgnoreMissing = true,
.pgFileSize = 9, .pgFileSize = 9,
.pgFileCopyExactSize = true, .pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), .pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false, .pgFileChecksumPage = false,
.manifestFile = pgFile, .manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false, .manifestFileHasReference = false,
}; };
@ -1152,7 +1161,7 @@ testRun(void)
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, true, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"file in repo only, checksum in repo equal, ignoreMissing=true, no pageChecksum, delta, no hasReference"); "file in repo only, checksum in repo equal, ignoreMissing=true, no pageChecksum, delta, no hasReference");
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy=repo=0 size"); TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy=repo=0 size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, "skip file"); TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, "skip file");
@ -1184,7 +1193,7 @@ testRun(void)
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeGz, 3, false, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeGz, 3, cipherTypeNone, NULL, fileList), 0),
"pg file exists, no checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference"); "pg file exists, no checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size"); TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 29, "repo compress size"); TEST_RESULT_UINT(result.repoSize, 29, "repo compress size");
@ -1209,6 +1218,7 @@ testRun(void)
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), .pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false, .pgFileChecksumPage = false,
.manifestFile = pgFile, .manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false, .manifestFileHasReference = false,
}; };
@ -1216,7 +1226,7 @@ testRun(void)
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeGz, 3, false, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeGz, 3, cipherTypeNone, NULL, fileList), 0),
"pg file & repo exists, match, checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference"); "pg file & repo exists, match, checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size"); TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 0, "repo size not calculated"); TEST_RESULT_UINT(result.repoSize, 0, "repo size not calculated");
@ -1253,7 +1263,7 @@ testRun(void)
// No prior checksum, no compression, no pageChecksum, no delta, no hasReference // No prior checksum, no compression, no pageChecksum, no delta, no hasReference
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, false, cipherTypeNone, NULL, fileList), 0), *(BackupFileResult *)lstGet(backupFile(repoFile, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
"zero-sized pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference"); "zero-sized pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy=repo=pgFile size 0"); TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy=repo=pgFile size 0");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file"); TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
@ -1305,7 +1315,7 @@ testRun(void)
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet( *(BackupFileResult *)lstGet(
backupFile(repoFile, compressTypeNone, 1, false, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0), backupFile(repoFile, compressTypeNone, 1, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
"pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference"); "pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy size set"); TEST_RESULT_UINT(result.copySize, 9, "copy size set");
TEST_RESULT_UINT(result.repoSize, 32, "repo size set"); TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
@ -1324,12 +1334,14 @@ testRun(void)
file = (BackupFile) file = (BackupFile)
{ {
.pgFile = pgFile, .pgFile = pgFile,
.pgFileDelta = true,
.pgFileIgnoreMissing = false, .pgFileIgnoreMissing = false,
.pgFileSize = 8, .pgFileSize = 8,
.pgFileCopyExactSize = true, .pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), .pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false, .pgFileChecksumPage = false,
.manifestFile = pgFile, .manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false, .manifestFileHasReference = false,
}; };
@ -1339,7 +1351,7 @@ testRun(void)
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet( *(BackupFileResult *)lstGet(
backupFile(repoFile, compressTypeNone, 1, true, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0), backupFile(repoFile, compressTypeNone, 1, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
"pg and repo file exists, pgFileMatch false, no ignoreMissing, no pageChecksum, delta, no hasReference"); "pg and repo file exists, pgFileMatch false, no ignoreMissing, no pageChecksum, delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 8, "copy size set"); TEST_RESULT_UINT(result.copySize, 8, "copy size set");
TEST_RESULT_UINT(result.repoSize, 32, "repo size set"); TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
@ -1364,6 +1376,7 @@ testRun(void)
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), .pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksumPage = false, .pgFileChecksumPage = false,
.manifestFile = pgFile, .manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false, .manifestFileHasReference = false,
}; };
@ -1372,7 +1385,7 @@ testRun(void)
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet( *(BackupFileResult *)lstGet(
backupFile(repoFile, compressTypeNone, 0, false, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0), backupFile(repoFile, compressTypeNone, 0, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
"pg and repo file exists, checksum mismatch, no ignoreMissing, no pageChecksum, no delta, no hasReference"); "pg and repo file exists, checksum mismatch, no ignoreMissing, no pageChecksum, no delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, "copy size set"); TEST_RESULT_UINT(result.copySize, 9, "copy size set");
TEST_RESULT_UINT(result.repoSize, 32, "repo size set"); TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
@ -1397,6 +1410,7 @@ testRun(void)
.pgFileChecksum = STRDEF("1234567890123456789012345678901234567890"), .pgFileChecksum = STRDEF("1234567890123456789012345678901234567890"),
.pgFileChecksumPage = false, .pgFileChecksumPage = false,
.manifestFile = pgFile, .manifestFile = pgFile,
.manifestFileResume = true,
.manifestFileHasReference = false, .manifestFileHasReference = false,
}; };
@ -1405,7 +1419,7 @@ testRun(void)
TEST_ASSIGN( TEST_ASSIGN(
result, result,
*(BackupFileResult *)lstGet( *(BackupFileResult *)lstGet(
backupFile(repoFile, compressTypeNone, 0, false, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0), backupFile(repoFile, compressTypeNone, 0, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
"backup file"); "backup file");
TEST_RESULT_UINT(result.copySize, 9, "copy size set"); TEST_RESULT_UINT(result.copySize, 9, "copy size set");

69
test/src/module/info/manifestTest.c
View File

@ -45,6 +45,15 @@ testRun(void)
"backup-timestamp-stop=0\n" \ "backup-timestamp-stop=0\n" \
"backup-type=\"full\"\n" "backup-type=\"full\"\n"
#define TEST_MANIFEST_HEADER_BUNDLE \
"[backup]\n" \
"backup-bundle=true\n" \
"backup-label=null\n" \
"backup-timestamp-copy-start=0\n" \
"backup-timestamp-start=0\n" \
"backup-timestamp-stop=0\n" \
"backup-type=\"full\"\n"
#define TEST_MANIFEST_DB_90 \ #define TEST_MANIFEST_DB_90 \
"\n" \ "\n" \
"[backup:db]\n" \ "[backup:db]\n" \
@ -801,7 +810,7 @@ testRun(void)
// pg_wal not ignored // pg_wal not ignored
TEST_ASSIGN( TEST_ASSIGN(
manifest, manifest,
manifestNewBuild(storagePg, PG_VERSION_13, hrnPgCatalogVersion(PG_VERSION_13), false, false, false, NULL, NULL), manifestNewBuild(storagePg, PG_VERSION_13, hrnPgCatalogVersion(PG_VERSION_13), false, false, true, NULL, NULL),
"build manifest"); "build manifest");
contentSave = bufNew(0); contentSave = bufNew(0);
@ -809,7 +818,7 @@ testRun(void)
TEST_RESULT_STR( TEST_RESULT_STR(
strNewBuf(contentSave), strNewBuf(contentSave),
strNewBuf(harnessInfoChecksumZ( strNewBuf(harnessInfoChecksumZ(
TEST_MANIFEST_HEADER TEST_MANIFEST_HEADER_BUNDLE
TEST_MANIFEST_DB_13 TEST_MANIFEST_DB_13
TEST_MANIFEST_OPTION_ALL TEST_MANIFEST_OPTION_ALL
"\n" "\n"
@ -1050,23 +1059,23 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/BOGUS"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/BOGUS"), .copy = true, .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .copy = true, .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182861, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .copy = true, .size = 55, .sizeRepo = 55, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .copy = true, .size = 4, .sizeRepo = 4,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
} }
OBJ_NEW_END(); OBJ_NEW_END();
@ -1130,13 +1139,26 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .copy = true, .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
// Zero-length file without the copy flag which will appear to come from a bundled backup
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-bundle"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
// Zero-length file with the copy flag which will appear to come from a non-bundled backup (so will get a reference)
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-normal"), .copy = true, .size = 0, .sizeRepo = 0,
.timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"),
.checksumSha1 = HASH_TYPE_SHA1_ZERO});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .copy = true, .size = 4, .sizeRepo = 4,
.timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifestPrior, manifestPrior,
@ -1144,6 +1166,17 @@ testRun(void)
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"), .reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"}); .checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-bundle"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-normal"), .size = 0, .sizeRepo = 0,
.timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"),
.checksumSha1 = HASH_TYPE_SHA1_ZERO});
TEST_RESULT_VOID(manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, NULL), "incremental manifest"); TEST_RESULT_VOID(manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, NULL), "incremental manifest");
@ -1160,6 +1193,8 @@ testRun(void)
"pg_data={\"path\":\"/pg\",\"type\":\"path\"}\n" "pg_data={\"path\":\"/pg\",\"type\":\"path\"}\n"
"\n" "\n"
"[target:file]\n" "[target:file]\n"
"pg_data/FILE0-bundle={\"size\":0,\"timestamp\":1482182860}\n"
"pg_data/FILE0-normal={\"reference\":\"20190101-010101F\",\"size\":0,\"timestamp\":1482182860}\n"
"pg_data/FILE1={\"checksum\":\"aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd\"," "pg_data/FILE1={\"checksum\":\"aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd\","
"\"reference\":\"20190101-010101F_20190202-010101D\",\"size\":4,\"timestamp\":1482182860}\n" "\"reference\":\"20190101-010101F_20190202-010101D\",\"size\":4,\"timestamp\":1482182860}\n"
"pg_data/PG_VERSION={\"checksum\":\"aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd\",\"reference\":\"20190101-010101F\"," "pg_data/PG_VERSION={\"checksum\":\"aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd\",\"reference\":\"20190101-010101F\","
@ -1181,7 +1216,7 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182859, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .copy = true, .size = 4, .sizeRepo = 4, .timestamp = 1482182859,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
// Clear prior manifest and add a single file with later timestamp and checksum error // Clear prior manifest and add a single file with later timestamp and checksum error
@ -1192,7 +1227,7 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifestPrior, manifestPrior,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .copy = true, .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"), .reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", .checksumPage = true, .checksumPageError = true, .checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", .checksumPage = true, .checksumPageError = true,
.checksumPageErrorList = jsonFromVar(varNewVarLst(checksumPageErrorList))}); .checksumPageErrorList = jsonFromVar(varNewVarLst(checksumPageErrorList))});
@ -1234,18 +1269,18 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .copy = true, .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .copy = true, .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifestPrior, manifestPrior,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .copy = true, .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"), .reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"}); .checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
@ -1304,7 +1339,7 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .copy = true, .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifest->pub.data.backupOptionOnline = BOOL_TRUE_VAR; manifest->pub.data.backupOptionOnline = BOOL_TRUE_VAR;
@ -1609,6 +1644,7 @@ testRun(void)
ManifestFile file = manifestFileFind(manifest, STRDEF("pg_data/postgresql.conf")); ManifestFile file = manifestFileFind(manifest, STRDEF("pg_data/postgresql.conf"));
file.checksumSha1[0] = '\0'; file.checksumSha1[0] = '\0';
file.sizeRepo = 0; file.sizeRepo = 0;
file.resume = true;
manifestFileUpdate(manifest, &file); manifestFileUpdate(manifest, &file);
file = manifestFileFind(manifest, STRDEF("pg_data/base/32768/33000.32767")); file = manifestFileFind(manifest, STRDEF("pg_data/base/32768/33000.32767"));
@ -1629,6 +1665,7 @@ testRun(void)
// Undo changes made to files // Undo changes made to files
file = manifestFileFind(manifest, STRDEF("pg_data/postgresql.conf")); file = manifestFileFind(manifest, STRDEF("pg_data/postgresql.conf"));
TEST_RESULT_BOOL(file.resume, true, "resume is set");
memcpy(file.checksumSha1, "184473f470864e067ee3a22e64b47b0a1c356f29", HASH_TYPE_SHA1_SIZE_HEX + 1); memcpy(file.checksumSha1, "184473f470864e067ee3a22e64b47b0a1c356f29", HASH_TYPE_SHA1_SIZE_HEX + 1);
file.sizeRepo = 4457; file.sizeRepo = 4457;
manifestFileUpdate(manifest, &file); manifestFileUpdate(manifest, &file);