db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-01-30 05:39:12 +02:00
Code Issues Releases Activity

Store manifest checksums in memory more efficiently.

Browse Source 
Manifest checksums were stored as hex-encoded strings due to legacy compatibility with Perl. Storing the checksums as binary in memory uses half the space and avoids many conversions.

There is no change to the on-disk manifest format which stores the checksum as a hex-encoded string.
This commit is contained in:
David Steele 2022-12-20 16:35:27 +07:00 committed by GitHub
parent 77c721eb63
commit 2ab845e263
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 219 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
doc/xml/release.xml
View File

@ -28,6 +28,24 @@
<p>Remove support for <proper>PostgreSQL</proper> <id>9.0</id>/<id>9.1</id>/<id>9.2</id>.</p>
</release-item>
</release-improvement-list>
<release-development-list>
<release-item>
<commit subject="Rename EncodeType enum to EncodingType."/>
<commit subject="Add hex encode/decoding to decode module."/>
<commit subject="Use smaller type for decode tables."/>
<commit subject="Store manifest checksums in memory more efficiently.">
<github-pull-request id="1953"/>
</commit>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="stefan.fercot"/>
</release-item-contributor-list>
<p>Store manifest checksums in memory more efficiently.</p>
</release-item>
</release-development-list>
</release-core-list>
</release>

23
src/command/backup/backup.c
View File

@ -566,7 +566,7 @@ backupResumeClean(
if (fileResume.reference != NULL)
removeReason = "reference in resumed manifest";
else if (fileResume.checksumSha1[0] == '\0')
else if (fileResume.checksumSha1 == NULL)
removeReason = "no checksum in resumed manifest";
else if (file.size != fileResume.size)
removeReason = "mismatched size";
@ -581,7 +581,7 @@ backupResumeClean(
ASSERT(file.bundleId == 0);
file.sizeRepo = fileResume.sizeRepo;
memcpy(file.checksumSha1, fileResume.checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
file.checksumSha1 = fileResume.checksumSha1;
file.checksumPage = fileResume.checksumPage;
file.checksumPageError = fileResume.checksumPageError;
file.checksumPageErrorList = fileResume.checksumPageErrorList;
@ -964,13 +964,9 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti
.size = strSize(content),
.sizeRepo = pckReadU64P(ioFilterGroupResultP(filterGroup, SIZE_FILTER_TYPE)),
.timestamp = timestamp,
.checksumSha1 = bufPtr(pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))),
};
memcpy(
file.checksumSha1,
strZ(strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))),
HASH_TYPE_SHA1_SIZE_HEX + 1);
manifestFileAdd(manifest, &file);
LOG_DETAIL_FMT("wrote '%s' file returned from backup stop function", strZ(name));
@ -1168,7 +1164,7 @@ backupJobResult(
const uint64_t copySize = pckReadU64P(jobResult);
const uint64_t bundleOffset = pckReadU64P(jobResult);
const uint64_t repoSize = pckReadU64P(jobResult);
const String *const copyChecksum = pckReadStrP(jobResult);
const Buffer *const copyChecksum = pckReadBinP(jobResult);
PackRead *const checksumPageResult = pckReadPackReadP(jobResult);
// Increment backup copy progress
@ -1191,7 +1187,8 @@ backupJobResult(
logProgress, "%s, %u.%02u%%", strZ(strSizeFormat(copySize)), percentComplete / 100, percentComplete % 100);
// Format log checksum
const String *const logChecksum = copySize != 0 ? strNewFmt(" checksum %s", strZ(copyChecksum)) : EMPTY_STR;
const String *const logChecksum = copySize != 0 ?
strNewFmt(" checksum %s", strZ(strNewEncode(encodingHex, copyChecksum))) : EMPTY_STR;
// If the file is in a prior backup and nothing changed, just log it
if (copyResult == backupCopyResultNoOp)
@ -1226,7 +1223,7 @@ backupJobResult(
" continue but this may be an issue unless the resumed backup path in the repository is known to be"
" corrupted.\n"
"NOTE: this does not indicate a problem with the PostgreSQL page checksums.",
strZ(file.name), file.checksumSha1);
strZ(file.name), strZ(strNewEncode(encodingHex, BUF(file.checksumSha1, HASH_TYPE_SHA1_SIZE))));
}
LOG_DETAIL_PID_FMT(processId, "backup file %s (%s)%s", strZ(fileLog), strZ(logProgress), strZ(logChecksum));
@ -1307,7 +1304,7 @@ backupJobResult(
// Update file info and remove any reference to the file's existence in a prior backup
file.size = copySize;
file.sizeRepo = repoSize;
memcpy(file.checksumSha1, strZ(copyChecksum), HASH_TYPE_SHA1_SIZE_HEX + 1);
file.checksumSha1 = bufPtrConst(copyChecksum);
file.reference = NULL;
file.checksumPageError = checksumPageError;
file.checksumPageErrorList = checksumPageErrorList != NULL ?
@ -1718,7 +1715,7 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
pckWriteBoolP(param, !strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)));
pckWriteU64P(param, file.size);
pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file.name));
pckWriteStrP(param, file.checksumSha1[0] != 0 ? STR(file.checksumSha1) : NULL);
pckWriteBinP(param, file.checksumSha1 != NULL ? BUF(file.checksumSha1, HASH_TYPE_SHA1_SIZE) : NULL);
pckWriteBoolP(param, file.checksumPage);
pckWriteStrP(param, file.name);
pckWriteBoolP(param, file.resume);
@ -2096,7 +2093,7 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif
.timestamp = manifestData(manifest)->backupTimestampStop,
};
memcpy(file.checksumSha1, strZ(strSubN(archiveFile, 25, 40)), HASH_TYPE_SHA1_SIZE_HEX + 1);
file.checksumSha1 = bufPtr(bufNewDecode(encodingHex, strSubN(archiveFile, 25, 40)));
manifestFileAdd(manifest, &file);
}

21
src/command/backup/file.c
View File

@ -94,12 +94,12 @@ backupFile(
// If the pg file exists check the checksum/size
if (ioReadDrain(read))
{
const String *const pgTestChecksum = strNewEncode(
encodingHex, pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE)));
const Buffer *const pgTestChecksum = pckReadBinP(
ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE));
uint64_t pgTestSize = pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(read), SIZE_FILTER_TYPE));
// Does the pg file match?
if (file->pgFileSize == pgTestSize && strEq(file->pgFileChecksum, pgTestChecksum))
if (file->pgFileSize == pgTestSize && bufEq(file->pgFileChecksum, pgTestChecksum))
{
pgFileMatch = true;
@ -110,7 +110,7 @@ backupFile(
{
fileResult->backupCopyResult = backupCopyResultNoOp;
fileResult->copySize = pgTestSize;
fileResult->copyChecksum = strDup(pgTestChecksum);
fileResult->copyChecksum = bufDup(pgTestChecksum);
}
MEM_CONTEXT_END();
}
@ -160,18 +160,18 @@ backupFile(
ioReadDrain(read);
// Test checksum/size
const String *const pgTestChecksum = strNewEncode(
encodingHex, pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE)));
const Buffer *const pgTestChecksum = pckReadBinP(
ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE));
uint64_t pgTestSize = pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(read), SIZE_FILTER_TYPE));
// No need to recopy if checksum/size match
if (file->pgFileSize == pgTestSize && strEq(file->pgFileChecksum, pgTestChecksum))
if (file->pgFileSize == pgTestSize && bufEq(file->pgFileChecksum, pgTestChecksum))
{
MEM_CONTEXT_BEGIN(lstMemContext(result))
{
fileResult->backupCopyResult = backupCopyResultChecksum;
fileResult->copySize = pgTestSize;
fileResult->copyChecksum = strDup(pgTestChecksum);
fileResult->copyChecksum = bufDup(pgTestChecksum);
}
MEM_CONTEXT_END();
}
@ -275,9 +275,8 @@ backupFile(
fileResult->copySize = pckReadU64P(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE, .idx = 0));
fileResult->bundleOffset = bundleOffset;
fileResult->copyChecksum = strNewEncode(
encodingHex,
pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE)));
fileResult->copyChecksum = pckReadBinP(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE));
fileResult->repoSize = pckReadU64P(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE, .idx = 1));

4
src/command/backup/file.h
View File

@ -31,7 +31,7 @@ typedef struct BackupFile
bool pgFileIgnoreMissing; // Ignore missing pg file
uint64_t pgFileSize; // Expected pg file size
bool pgFileCopyExactSize; // Copy only pg expected size
const String *pgFileChecksum; // Expected pg file checksum
const Buffer *pgFileChecksum; // Expected pg file checksum
bool pgFileChecksumPage; // Validate page checksums?
const String *manifestFile; // Repo file
bool manifestFileResume; // Checksum repo file before copying
@ -43,7 +43,7 @@ typedef struct BackupFileResult
const String *manifestFile; // Manifest file
BackupCopyResult backupCopyResult;
uint64_t copySize;
String *copyChecksum;
Buffer *copyChecksum;
uint64_t bundleOffset; // Offset in bundle if any
uint64_t repoSize;
Pack *pageChecksumResult;

4
src/command/backup/protocol.c
View File

@ -44,7 +44,7 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
file.pgFileIgnoreMissing = pckReadBoolP(param);
file.pgFileSize = pckReadU64P(param);
file.pgFileCopyExactSize = pckReadBoolP(param);
file.pgFileChecksum = pckReadStrP(param);
file.pgFileChecksum = pckReadBinP(param);
file.pgFileChecksumPage = pckReadBoolP(param);
file.manifestFile = pckReadStrP(param);
file.manifestFileResume = pckReadBoolP(param);
@ -69,7 +69,7 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
pckWriteU64P(resultPack, fileResult->copySize);
pckWriteU64P(resultPack, fileResult->bundleOffset);
pckWriteU64P(resultPack, fileResult->repoSize);
pckWriteStrP(resultPack, fileResult->copyChecksum);
pckWriteBinP(resultPack, fileResult->copyChecksum);
pckWritePackP(resultPack, fileResult->pageChecksumResult);
}

12
src/command/restore/file.c
View File

@ -119,11 +119,9 @@ List *restoreFile(
// If the checksum is the same (or file is zero size) then no need to copy the file
if (file->size == 0 ||
strEq(
bufEq(
file->checksum,
strNewEncode(
encodingHex,
pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE)))))
pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE))))
{
// If the hash/size are now the same but the time is not, then set the time back to the backup
// time. This helps with unit testing, but also presents a pristine version of the database
@ -269,16 +267,14 @@ List *restoreFile(
storageReadFree(repoFileRead);
// Validate checksum
if (!strEq(
file->checksum,
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))))
if (!bufEq(file->checksum, pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))))
{
THROW_FMT(
ChecksumError,
"error restoring '%s': actual checksum '%s' does not match expected checksum '%s'", strZ(file->name),
strZ(
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))),
strZ(file->checksum));
strZ(strNewEncode(encodingHex, file->checksum)));
}
}
}

2
src/command/restore/file.h
View File

@ -24,7 +24,7 @@ Functions
typedef struct RestoreFile
{
const String *name; // File to restore
const String *checksum; // Expected checksum
const Buffer *checksum; // Expected checksum
uint64_t size; // Expected size
time_t timeModified; // Original modification time
mode_t mode; // Original mode

2
src/command/restore/protocol.c
View File

@ -41,7 +41,7 @@ restoreFileProtocol(PackRead *const param, ProtocolServer *const server)
while (!pckReadNullP(param))
{
RestoreFile file = {.name = pckReadStrP(param)};
file.checksum = pckReadStrP(param);
file.checksum = pckReadBinP(param);
file.size = pckReadU64P(param);
file.timeModified = pckReadTimeP(param);
file.mode = pckReadModeP(param);

4
src/command/restore/restore.c
View File

@ -2196,7 +2196,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
// If not zero-length add the checksum
if (file.size != 0 && !zeroed)
strCatFmt(log, " checksum %s", file.checksumSha1);
strCatFmt(log, " checksum %s", strZ(strNewEncode(encodingHex, BUF(file.checksumSha1, HASH_TYPE_SHA1_SIZE))));
LOG_DETAIL_PID(protocolParallelJobProcessId(job), strZ(log));
}
@ -2319,7 +2319,7 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
}
pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file.name));
pckWriteStrP(param, STR(file.checksumSha1));
pckWriteBinP(param, BUF(file.checksumSha1, HASH_TYPE_SHA1_SIZE));
pckWriteU64P(param, file.size);
pckWriteTimeP(param, file.timestamp);
pckWriteModeP(param, file.mode);

8
src/command/verify/file.c
View File

@ -18,14 +18,14 @@ Verify File
VerifyResult
verifyFile(
const String *const filePathName, const uint64_t offset, const Variant *const limit, const CompressType compressType,
const String *const fileChecksum, const uint64_t fileSize, const String *const cipherPass)
const Buffer *const fileChecksum, const uint64_t fileSize, const String *const cipherPass)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, filePathName); // Fully qualified file name
FUNCTION_LOG_PARAM(UINT64, offset); // Offset to read in file
FUNCTION_LOG_PARAM(VARIANT, limit); // Limit to read from file
FUNCTION_LOG_PARAM(ENUM, compressType); // Compression type
FUNCTION_LOG_PARAM(STRING, fileChecksum); // Checksum for the file
FUNCTION_LOG_PARAM(BUFFER, fileChecksum); // Checksum for the file
FUNCTION_LOG_PARAM(UINT64, fileSize); // Size of file
FUNCTION_TEST_PARAM(STRING, cipherPass); // Password to access the repo file if encrypted
FUNCTION_LOG_END();
@ -65,9 +65,7 @@ verifyFile(
if (ioReadDrain(read))
{
// Validate checksum
if (!strEq(
fileChecksum,
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))))
if (!bufEq(fileChecksum, pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))))
{
result = verifyChecksumMismatch;
}

2
src/command/verify/file.h
View File

@ -24,7 +24,7 @@ Functions
***********************************************************************************************************************************/
// Verify a file in the pgBackRest repository
VerifyResult verifyFile(
const String *filePathName, uint64_t offset, const Variant *limit, CompressType compressType, const String *fileChecksum,
const String *filePathName, uint64_t offset, const Variant *limit, CompressType compressType, const Buffer *fileChecksum,
uint64_t fileSize, const String *cipherPass);
#endif

2
src/command/verify/protocol.c
View File

@ -39,7 +39,7 @@ verifyFileProtocol(PackRead *const param, ProtocolServer *const server)
}
const CompressType compressType = (CompressType)pckReadU32P(param);
const String *const fileChecksum = pckReadStrP(param);
const Buffer *const fileChecksum = pckReadBinP(param);
const uint64_t fileSize = pckReadU64P(param);
const String *const cipherPass = pckReadStrP(param);

7
src/command/verify/verify.c
View File

@ -741,7 +741,8 @@ verifyArchive(VerifyJobData *const jobData)
const String *fileName = strLstGet(jobData->walFileList, 0);
const String *filePathName = strNewFmt(
STORAGE_REPO_ARCHIVE "/%s/%s/%s", strZ(archiveResult->archiveId), strZ(walPath), strZ(fileName));
String *checksum = strSubN(fileName, WAL_SEGMENT_NAME_SIZE + 1, HASH_TYPE_SHA1_SIZE_HEX);
Buffer *const checksum = bufNewDecode(
encodingHex, strSubN(fileName, WAL_SEGMENT_NAME_SIZE + 1, HASH_TYPE_SHA1_SIZE_HEX));
// Set up the job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
@ -750,7 +751,7 @@ verifyArchive(VerifyJobData *const jobData)
pckWriteStrP(param, filePathName);
pckWriteBoolP(param, false);
pckWriteU32P(param, compressTypeFromName(filePathName));
pckWriteStrP(param, checksum);
pckWriteBinP(param, checksum);
pckWriteU64P(param, archiveResult->pgWalInfo.size);
pckWriteStrP(param, jobData->walCipherPass);
@ -995,7 +996,7 @@ verifyBackup(VerifyJobData *const jobData)
pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType);
// If the checksum is not present in the manifest, it will be calculated by manifest load
pckWriteStrP(param, STR(fileData.checksumSha1));
pckWriteBinP(param, BUF(fileData.checksumSha1, HASH_TYPE_SHA1_SIZE));
pckWriteU64P(param, fileData.size);
pckWriteStrP(param, jobData->backupCipherPass);

5
src/common/crypto/hash.c
View File

@ -20,7 +20,10 @@ Cryptographic Hash
/***********************************************************************************************************************************
Hashes for zero-length files (i.e., seed value)
***********************************************************************************************************************************/
STRING_EXTERN(HASH_TYPE_SHA1_ZERO_STR, HASH_TYPE_SHA1_ZERO);
static uint8_t hashTypeSha1Zero[HASH_TYPE_SHA1_SIZE] =
{0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09};
const Buffer *const HASH_TYPE_SHA1_ZERO_BUF = BUF(hashTypeSha1Zero, sizeof(hashTypeSha1Zero));
STRING_EXTERN(HASH_TYPE_SHA256_ZERO_STR, HASH_TYPE_SHA256_ZERO);
/***********************************************************************************************************************************

2
src/common/crypto/hash.h
View File

@ -20,7 +20,7 @@ Hashes for zero-length files (i.e., starting hash)
***********************************************************************************************************************************/
#define HASH_TYPE_MD5_ZERO "d41d8cd98f00b204e9800998ecf8427e"
#define HASH_TYPE_SHA1_ZERO "da39a3ee5e6b4b0d3255bfef95601890afd80709"
STRING_DECLARE(HASH_TYPE_SHA1_ZERO_STR);
BUFFER_DECLARE(HASH_TYPE_SHA1_ZERO_BUF);
#define HASH_TYPE_SHA256_ZERO \
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
STRING_DECLARE(HASH_TYPE_SHA256_ZERO_STR);

44
src/info/manifest.c
View File

@ -94,6 +94,7 @@ static time_t manifestPackBaseTime = -1;
// Flags used to reduce the size of packed data. They should be ordered from most to least likely and can be reordered at will.
typedef enum
{
manifestFilePackFlagChecksum,
manifestFilePackFlagReference,
manifestFilePackFlagBundle,
manifestFilePackFlagCopy,
@ -127,6 +128,9 @@ manifestFilePack(const Manifest *const manifest, const ManifestFile *const file)
// Flags
uint64_t flag = 0;
if (file->checksumSha1 != NULL)
flag |= 1 << manifestFilePackFlagChecksum;
if (file->copy)
flag |= 1 << manifestFilePackFlagCopy;
@ -178,8 +182,11 @@ manifestFilePack(const Manifest *const manifest, const ManifestFile *const file)
cvtUInt64ToVarInt128(cvtInt64ToZigZag(manifestPackBaseTime - file->timestamp), buffer, &bufferPos, sizeof(buffer));
// SHA1 checksum
strcpy((char *)buffer + bufferPos, file->checksumSha1);
bufferPos += HASH_TYPE_SHA1_SIZE_HEX + 1;
if (file->checksumSha1 != NULL)
{
memcpy((uint8_t *)buffer + bufferPos, file->checksumSha1, HASH_TYPE_SHA1_SIZE);
bufferPos += HASH_TYPE_SHA1_SIZE;
}
// Reference
if (file->reference != NULL)
@ -277,8 +284,11 @@ manifestFileUnpack(const Manifest *const manifest, const ManifestFilePack *const
result.checksumPage = (flag >> manifestFilePackFlagChecksumPage) & 1;
// SHA1 checksum
memcpy(result.checksumSha1, (const uint8_t *)filePack + bufferPos, HASH_TYPE_SHA1_SIZE_HEX + 1);
bufferPos += HASH_TYPE_SHA1_SIZE_HEX + 1;
if (flag & (1 << manifestFilePackFlagChecksum))
{
result.checksumSha1 = (const uint8_t *)filePack + bufferPos;
bufferPos += HASH_TYPE_SHA1_SIZE;
}
// Reference
if (flag & (1 << manifestFilePackFlagReference))
@ -926,7 +936,7 @@ manifestBuildInfo(
if (info->size == 0 && buildData->manifest->pub.data.bundle)
{
file.copy = false;
memcpy(file.checksumSha1, HASH_TYPE_SHA1_ZERO, HASH_TYPE_SHA1_SIZE_HEX + 1);
file.checksumSha1 = bufPtrConst(HASH_TYPE_SHA1_ZERO_BUF);
}
// Determine if this file should be page checksummed
@ -1512,7 +1522,7 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
if (file.copy && file.size == filePrior.size && (delta || file.size == 0 || file.timestamp == filePrior.timestamp))
{
file.sizeRepo = filePrior.sizeRepo;
memcpy(file.checksumSha1, filePrior.checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
file.checksumSha1 = filePrior.checksumSha1;
file.reference = filePrior.reference != NULL ? filePrior.reference : manifestPrior->pub.data.backupLabel;
file.checksumPage = filePrior.checksumPage;
file.checksumPageError = filePrior.checksumPageError;
@ -1814,7 +1824,7 @@ manifestLoadCallback(void *callbackData, const String *const section, const Stri
// The checksum might not exist if this is a partial save that was done during the backup to preserve checksums for already
// backed up files
if (jsonReadKeyExpectStrId(json, MANIFEST_KEY_CHECKSUM))
memcpy(file.checksumSha1, strZ(jsonReadStr(json)), HASH_TYPE_SHA1_SIZE_HEX + 1);
file.checksumSha1 = bufPtr(bufNewDecode(encodingHex, jsonReadStr(json)));
// Page checksum errors
if (jsonReadKeyExpectZ(json, MANIFEST_KEY_CHECKSUM_PAGE))
@ -1868,7 +1878,7 @@ manifestLoadCallback(void *callbackData, const String *const section, const Stri
// If file size is zero then assign the static zero hash
if (file.size == 0)
memcpy(file.checksumSha1, HASH_TYPE_SHA1_ZERO, HASH_TYPE_SHA1_SIZE_HEX + 1);
file.checksumSha1 = bufPtrConst(HASH_TYPE_SHA1_ZERO_BUF);
// Timestamp is required so error if it is not present
if (jsonReadKeyExpectStrId(json, MANIFEST_KEY_TIMESTAMP))
@ -2524,8 +2534,12 @@ manifestSaveCallback(void *const callbackData, const String *const sectionNext,
// Save if the file size is not zero and the checksum exists. The checksum might not exist if this is a partial
// save performed during a backup.
if (file.size != 0 && file.checksumSha1[0] != 0)
jsonWriteZ(jsonWriteKeyStrId(json, MANIFEST_KEY_CHECKSUM), file.checksumSha1);
if (file.size != 0 && file.checksumSha1 != NULL)
{
jsonWriteStr(
jsonWriteKeyStrId(json, MANIFEST_KEY_CHECKSUM),
strNewEncode(encodingHex, BUF(file.checksumSha1, HASH_TYPE_SHA1_SIZE)));
}
if (file.checksumPage)
{
@ -2709,15 +2723,19 @@ manifestValidate(Manifest *this, bool strict)
const ManifestFile file = manifestFile(this, fileIdx);
// All files must have a checksum
if (file.checksumSha1[0] == '\0')
if (file.checksumSha1 == NULL)
strCatFmt(error, "\nmissing checksum for file '%s'", strZ(file.name));
// These are strict checks to be performed only after a backup and before the final manifest save
if (strict)
{
// Zero-length files must have a specific checksum
if (file.size == 0 && !strEqZ(HASH_TYPE_SHA1_ZERO_STR, file.checksumSha1))
strCatFmt(error, "\ninvalid checksum '%s' for zero size file '%s'", file.checksumSha1, strZ(file.name));
if (file.size == 0 && !bufEq(HASH_TYPE_SHA1_ZERO_BUF, BUF(file.checksumSha1, HASH_TYPE_SHA1_SIZE)))
{
strCatFmt(
error, "\ninvalid checksum '%s' for zero size file '%s'",
strZ(strNewEncode(encodingHex, BUF(file.checksumSha1, HASH_TYPE_SHA1_SIZE))), strZ(file.name));
}
// Non-zero size files must have non-zero repo size
if (file.sizeRepo == 0 && file.size != 0)

2
src/info/manifest.h
View File

@ -105,7 +105,7 @@ typedef struct ManifestFile
bool checksumPage:1; // Does this file have page checksums?
bool checksumPageError:1; // Is there an error in the page checksum?
mode_t mode; // File mode
char checksumSha1[HASH_TYPE_SHA1_SIZE_HEX + 1]; // SHA1 checksum
const uint8_t *checksumSha1; // SHA1 checksum
const String *checksumPageErrorList; // List of page checksum errors if there are any
const String *user; // User name
const String *group; // Group name

90
test/src/module/command/backupTest.c
View File

@ -121,13 +121,12 @@ testBackupValidateList(
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(hashTypeSha1));
uint64_t size = bufUsed(storageGetP(read));
const String *checksum = strNewEncode(
encodingHex,
pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE)));
const Buffer *const checksum = pckReadBinP(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE));
strCatFmt(result, ", s=%" PRIu64, size);
if (!strEqZ(checksum, file.checksumSha1))
if (!bufEq(checksum, BUF(file.checksumSha1, HASH_TYPE_SHA1_SIZE)))
THROW_FMT(AssertError, "'%s' checksum does match manifest", strZ(file.name));
// Test size and repo-size. If compressed then set the repo-size to size so it will not be in test output. Even
@ -159,7 +158,7 @@ testBackupValidateList(
strBeginsWith(
file.name, strNewFmt(MANIFEST_TARGET_PGDATA "/%s/", strZ(pgWalPath(manifestData->pgVersion)))))
{
file.checksumSha1[0] = '\0';
file.checksumSha1 = NULL;
}
strCatZ(result, "}\n");
@ -909,7 +908,8 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 9, "repo=pgFile size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum matches");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum matches");
TEST_RESULT_STR_Z(hrnPackToStr(result.pageChecksumResult), "2:bool:false, 3:bool:false", "pageChecksumResult");
TEST_STORAGE_EXISTS(storageRepoWrite(), strZ(backupPathFile), .remove = true, .comment = "check exists in repo, remove");
@ -939,7 +939,7 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 12, "copy size");
TEST_RESULT_UINT(result.repoSize, 12, "repo size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
TEST_RESULT_STR_Z(result.copyChecksum, "c3ae4687ea8ccd47bfdb190dbe7fd3b37545fdb9", "checksum");
TEST_RESULT_STR_Z(strNewEncode(encodingHex, result.copyChecksum), "c3ae4687ea8ccd47bfdb190dbe7fd3b37545fdb9", "checksum");
TEST_RESULT_STR_Z(hrnPackToStr(result.pageChecksumResult), "2:bool:false, 3:bool:false", "page checksum");
TEST_STORAGE_GET(storageRepo(), strZ(backupPathFile), "atestfile###");
@ -955,7 +955,7 @@ testRun(void)
.pgFileIgnoreMissing = false,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksum = bufNewDecode(encodingHex, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileHasReference = true,
@ -971,7 +971,8 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 9, "copy size set");
TEST_RESULT_UINT(result.repoSize, 0, "repo size not set since already exists in repo");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultNoOp, "noop file");
TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum matches");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum matches");
TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
TEST_STORAGE_GET(storageRepo(), strZ(backupPathFile), "atestfile###", .comment = "file not modified");
@ -987,7 +988,7 @@ testRun(void)
.pgFileIgnoreMissing = false,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("1234567890123456789012345678901234567890"),
.pgFileChecksum = bufNewDecode(encodingHex, STRDEF("1234567890123456789012345678901234567890")),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileHasReference = true,
@ -1003,7 +1004,9 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum for file size 9");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67",
"copy checksum for file size 9");
TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
TEST_STORAGE_GET(storageRepo(), strZ(backupPathFile), "atestfile", .comment = "9 bytes copied");
@ -1019,7 +1022,7 @@ testRun(void)
.pgFileIgnoreMissing = false,
.pgFileSize = 9999999,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksum = bufNewDecode(encodingHex, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileHasReference = true,
@ -1035,7 +1038,8 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 12, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 12, "repo=pgFile size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
TEST_RESULT_STR_Z(result.copyChecksum, "c3ae4687ea8ccd47bfdb190dbe7fd3b37545fdb9", "copy checksum updated");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "c3ae4687ea8ccd47bfdb190dbe7fd3b37545fdb9", "copy checksum updated");
TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
TEST_STORAGE_GET(storageRepo(), strZ(backupPathFile), "atestfile###", .comment = "confirm contents");
@ -1051,7 +1055,7 @@ testRun(void)
.pgFileIgnoreMissing = false,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksum = bufNewDecode(encodingHex, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")),
.pgFileChecksumPage = false,
.manifestFile = STRDEF(BOGUS_STR),
.manifestFileResume = true,
@ -1071,7 +1075,9 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, "check recopy result");
TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum for file size 9");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67",
"copy checksum for file size 9");
TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
TEST_STORAGE_GET(
storageRepo(), strZ(backupPathFile), "atestfile###", .comment = "existing file with same name as pgFile not modified");
@ -1094,7 +1100,7 @@ testRun(void)
.pgFileIgnoreMissing = false,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksum = bufNewDecode(encodingHex, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
@ -1113,7 +1119,9 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, "recopy file");
TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum for file size 9");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67",
"copy checksum for file size 9");
TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
TEST_STORAGE_GET(storageRepo(), strZ(backupPathFile), "atestfile", .comment = "existing file recopied");
@ -1129,7 +1137,7 @@ testRun(void)
.pgFileIgnoreMissing = true,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksum = bufNewDecode(encodingHex, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
@ -1177,7 +1185,8 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 29, "repo compress size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum");
TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum result is NULL");
TEST_STORAGE_EXISTS(
storageRepo(), zNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strZ(backupLabel), strZ(pgFile)),
@ -1194,7 +1203,7 @@ testRun(void)
.pgFileIgnoreMissing = false,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksum = bufNewDecode(encodingHex, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
@ -1210,7 +1219,9 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
TEST_RESULT_UINT(result.repoSize, 0, "repo size not calculated");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultChecksum, "checksum file");
TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "compressed repo file checksum matches");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67",
"compressed repo file checksum matches");
TEST_STORAGE_EXISTS(
storageRepo(), zNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strZ(backupLabel), strZ(pgFile)),
.comment = "compressed file exists");
@ -1299,7 +1310,8 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 9, "copy size set");
TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum");
TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum NULL");
TEST_STORAGE_GET(
storageRepo(), strZ(backupPathFile), "atestfile", .cipherType = cipherTypeAes256Cbc,
@ -1317,7 +1329,7 @@ testRun(void)
.pgFileIgnoreMissing = false,
.pgFileSize = 8,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksum = bufNewDecode(encodingHex, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
@ -1335,7 +1347,9 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 8, "copy size set");
TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
TEST_RESULT_STR_Z(result.copyChecksum, "acc972a8319d4903b839c64ec217faa3e77b4fcb", "copy checksum for size passed");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "acc972a8319d4903b839c64ec217faa3e77b4fcb",
"copy checksum for size passed");
TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum NULL");
TEST_STORAGE_GET(
storageRepo(), strZ(backupPathFile), "atestfil", .cipherType = cipherTypeAes256Cbc,
@ -1352,7 +1366,7 @@ testRun(void)
.pgFileIgnoreMissing = false,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"),
.pgFileChecksum = bufNewDecode(encodingHex, STRDEF("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
@ -1369,7 +1383,8 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 9, "copy size set");
TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, "recopy file");
TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum");
TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum NULL");
TEST_STORAGE_GET(
storageRepoWrite(), strZ(backupPathFile), "atestfile", .cipherType = cipherTypeAes256Cbc,
@ -1386,7 +1401,7 @@ testRun(void)
.pgFileIgnoreMissing = false,
.pgFileSize = 9,
.pgFileCopyExactSize = true,
.pgFileChecksum = STRDEF("1234567890123456789012345678901234567890"),
.pgFileChecksum = bufNewDecode(encodingHex, STRDEF("1234567890123456789012345678901234567890")),
.pgFileChecksumPage = false,
.manifestFile = pgFile,
.manifestFileResume = true,
@ -1404,7 +1419,9 @@ testRun(void)
TEST_RESULT_UINT(result.copySize, 9, "copy size set");
TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultReCopy, "recopy file");
TEST_RESULT_STR_Z(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67", "copy checksum for size passed");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, result.copyChecksum), "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67",
"copy checksum for size passed");
TEST_RESULT_PTR(result.pageChecksumResult, NULL, "page checksum NULL");
TEST_STORAGE_GET(
storageRepo(), strZ(backupPathFile), "atestfile",
@ -2215,7 +2232,7 @@ testRun(void)
ManifestFilePack **const filePack = manifestFilePackFindInternal(manifestResume, STRDEF("pg_data/PG_VERSION"));
ManifestFile file = manifestFileUnpack(manifestResume, *filePack);
strcpy(file.checksumSha1, "06d06bb31b570b94d7b4325f511f853dbe771c21");
file.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("06d06bb31b570b94d7b4325f511f853dbe771c21")));
manifestFilePackUpdate(manifestResume, filePack, &file);
@ -2312,7 +2329,7 @@ testRun(void)
ManifestFilePack **const filePack = manifestFilePackFindInternal(manifestResume, STRDEF("pg_data/global/pg_control"));
ManifestFile file = manifestFileUnpack(manifestResume, *filePack);
file.checksumSha1[0] = 0;
file.checksumSha1 = NULL;
manifestFilePackUpdate(manifestResume, filePack, &file);
@ -2322,7 +2339,8 @@ testRun(void)
storageRepoWrite(), zNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/size-mismatch.gz", strZ(resumeLabel)));
manifestFileAdd(
manifestResume, &(ManifestFile){
.name = STRDEF("pg_data/size-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
.name = STRDEF("pg_data/size-mismatch"),
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))),
.size = 33});
// Time does not match between cluster and resume manifest
@ -2331,8 +2349,9 @@ testRun(void)
storageRepoWrite(), zNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch.gz", strZ(resumeLabel)));
manifestFileAdd(
manifestResume, &(ManifestFile){
.name = STRDEF("pg_data/time-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", .size = 4,
.timestamp = backupTimeStart - 1});
.name = STRDEF("pg_data/time-mismatch"),
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))),
.size = 4, .timestamp = backupTimeStart - 1});
// Size is zero in cluster and resume manifest. ??? We'd like to remove this requirement after the migration.
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "zero-size", .timeModified = backupTimeStart);
@ -2507,8 +2526,9 @@ testRun(void)
storageRepoWrite(), zNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch2.gz", strZ(resumeLabel)));
manifestFileAdd(
manifestResume, &(ManifestFile){
.name = STRDEF("pg_data/time-mismatch2"), .checksumSha1 = "984816fd329622876e14907634264e6f332e9fb3", .size = 4,
.timestamp = backupTimeStart});
.name = STRDEF("pg_data/time-mismatch2"),
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("984816fd329622876e14907634264e6f332e9fb3"))),
.size = 4, .timestamp = backupTimeStart});
// Links are always removed on resume
THROW_ON_SYS_ERROR(

75
test/src/module/command/restoreTest.c
View File

@ -177,7 +177,7 @@ testRun(void)
RestoreFile file =
{
.name = STRDEF("normal"),
.checksum = STRDEF("ffffffffffffffffffffffffffffffffffffffff"),
.checksum = bufNewDecode(encodingHex, STRDEF("ffffffffffffffffffffffffffffffffffffffff")),
.size = 7,
.timeModified = 1557432154,
.mode = 0600,
@ -2028,7 +2028,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("8dbabb96e032b8d9f1993c0e4b9141e71ade01a1")))});
HRN_STORAGE_PUT_Z(storageRepoIdxWrite(0), TEST_REPO_PATH PG_FILE_PGVERSION, PG_VERSION_94_STR "\n");
// Store the file also to the encrypted repo
@ -2169,7 +2169,7 @@ testRun(void)
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = bufPtrConst(HASH_TYPE_SHA1_ZERO_BUF)});
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP);
manifestFileAdd(
@ -2177,7 +2177,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/postgresql.conf"), .size = 10,
.timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "1a49a3c2240449fee1422e4afcf44d5b96378511"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("1a49a3c2240449fee1422e4afcf44d5b96378511")))});
HRN_STORAGE_PUT_Z(storageRepoWrite(), TEST_REPO_PATH "/postgresql.conf", "VALID_CONF");
manifestFileAdd(
@ -2185,7 +2185,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/postgresql.auto.conf"), .size = 15,
.timestamp = 1482182861, .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "37a0c84d42c3ec3d08c311cec2cef2a7ab55a7c3"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("37a0c84d42c3ec3d08c311cec2cef2a7ab55a7c3")))});
HRN_STORAGE_PUT_Z(storageRepoWrite(), TEST_REPO_PATH "/postgresql.auto.conf", "VALID_CONF_AUTO");
manifestFileAdd(
@ -2193,7 +2193,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/size-mismatch"), .size = 1,
.timestamp = 1482182861, .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "c032adc1ff629c9b66f22749ad667e6beadf144b"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("c032adc1ff629c9b66f22749ad667e6beadf144b")))});
HRN_STORAGE_PUT_Z(storageRepoWrite(), TEST_REPO_PATH "/size-mismatch", "X");
// pg_tblspc/1
@ -2228,7 +2228,8 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/1/16384/" PG_FILE_PGVERSION), .size = 4,
.timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("8dbabb96e032b8d9f1993c0e4b9141e71ade01a1")))});
HRN_STORAGE_PUT_Z(
storageRepoWrite(), STORAGE_REPO_BACKUP "/" TEST_LABEL "/" MANIFEST_TARGET_PGTBLSPC "/1/16384/" PG_FILE_PGVERSION,
PG_VERSION_94_STR "\n");
@ -2449,7 +2450,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL), .size = 8192, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "5e2b96c19c4f5c63a5afa2de504d29fe64a4c908"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("5e2b96c19c4f5c63a5afa2de504d29fe64a4c908")))});
HRN_STORAGE_PUT(storageRepoWrite(), TEST_REPO_PATH PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, fileBuffer);
// global/888
@ -2458,7 +2459,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/888"), .size = 0, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = HASH_TYPE_SHA1_ZERO});
.checksumSha1 = bufPtrConst(HASH_TYPE_SHA1_ZERO_BUF)});
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_PATH_GLOBAL "/888");
// global/999
@ -2467,7 +2468,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/999"), .size = 0, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = HASH_TYPE_SHA1_ZERO});
.checksumSha1 = bufPtrConst(HASH_TYPE_SHA1_ZERO_BUF)});
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_PATH_GLOBAL "/999");
// PG_VERSION
@ -2475,33 +2476,34 @@ testRun(void)
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 0,
.reference = NULL, .checksumSha1 = "2fafe15172578a19dbc196723bca6a4a8ad70da8"});
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 0, .reference = NULL,
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("2fafe15172578a19dbc196723bca6a4a8ad70da8")))});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "yyy"), .size = 3, .sizeRepo = 3, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 8,
.reference = NULL, .checksumSha1 = "186154712b2d5f6791d85b9a0987b98fa231779c"});
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 8, .reference = NULL,
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("186154712b2d5f6791d85b9a0987b98fa231779c")))});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "xxxxx"), .size = 5, .sizeRepo = 5, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 11,
.reference = NULL, .checksumSha1 = "9addbf544119efa4a64223b649750a510f0d463f"});
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 11, .reference = NULL,
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("9addbf544119efa4a64223b649750a510f0d463f")))});
// Set bogus sizeRepo and checksumSha1 to ensure this is not handled as a regular file
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "zero-length"), .size = 0, .sizeRepo = 1, .timestamp = 1482182866,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 16,
.reference = NULL, .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"});
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 16, .reference = NULL,
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "zz"), .size = 2, .sizeRepo = 2, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 17,
.reference = NULL, .checksumSha1 = "d7dacae2c968388960bf8970080a980ed5c5dcb7"});
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 17, .reference = NULL,
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("d7dacae2c968388960bf8970080a980ed5c5dcb7")))});
HRN_STORAGE_PUT_Z(
storageRepoWrite(), STORAGE_REPO_BACKUP "/" TEST_LABEL "/bundle/1",
PG_VERSION_96_STR "\n" PG_VERSION_96_STR "\nyyyxxxxxAzzA");
@ -2524,7 +2526,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 4,
.checksumSha1 = "2fafe15172578a19dbc196723bca6a4a8ad70da8"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("2fafe15172578a19dbc196723bca6a4a8ad70da8")))});
// base/1/2
fileBuffer = bufNew(8192);
@ -2536,7 +2538,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/2"), .size = 8192, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "4d7b2a36c5387decf799352a3751883b7ceb96aa"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("4d7b2a36c5387decf799352a3751883b7ceb96aa")))});
HRN_STORAGE_PUT(storageRepoWrite(), TEST_REPO_PATH "base/1/2", fileBuffer);
// base/1/10
@ -2551,7 +2553,8 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/10"), .size = 8192, .sizeRepo = 8192, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 1,
.reference = STRDEF(TEST_LABEL_FULL), .checksumSha1 = "28757c756c03c37aca13692cb719c18d1510c190"});
.reference = STRDEF(TEST_LABEL_FULL),
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("28757c756c03c37aca13692cb719c18d1510c190")))});
HRN_STORAGE_PUT(storageRepoWrite(), STORAGE_REPO_BACKUP "/" TEST_LABEL_FULL "/bundle/1", fileBuffer);
// base/1/20 and base/1/21
@ -2560,13 +2563,15 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/20"), .size = 1, .sizeRepo = 1, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 2, .bundleOffset = 1,
.reference = STRDEF(TEST_LABEL_DIFF), .checksumSha1 = "c032adc1ff629c9b66f22749ad667e6beadf144b"});
.reference = STRDEF(TEST_LABEL_DIFF),
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("c032adc1ff629c9b66f22749ad667e6beadf144b")))});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/21"), .size = 1, .sizeRepo = 1, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 2, .bundleOffset = 2,
.reference = STRDEF(TEST_LABEL_DIFF), .checksumSha1 = "e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98"});
.reference = STRDEF(TEST_LABEL_DIFF),
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98")))});
HRN_STORAGE_PUT_Z(storageRepoWrite(), STORAGE_REPO_BACKUP "/" TEST_LABEL_DIFF "/bundle/2", "aXb");
// base/1/30 and base/1/31
@ -2575,13 +2580,15 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/30"), .size = 1, .sizeRepo = 1, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 2, .bundleOffset = 1,
.reference = STRDEF(TEST_LABEL_INCR), .checksumSha1 = "c032adc1ff629c9b66f22749ad667e6beadf144b"});
.reference = STRDEF(TEST_LABEL_INCR),
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("c032adc1ff629c9b66f22749ad667e6beadf144b")))});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/31"), .size = 1, .sizeRepo = 1, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 2, .bundleOffset = 2,
.reference = STRDEF(TEST_LABEL_INCR), .checksumSha1 = "e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98"});
.reference = STRDEF(TEST_LABEL_INCR),
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98")))});
HRN_STORAGE_PUT_Z(storageRepoWrite(), STORAGE_REPO_BACKUP "/" TEST_LABEL_INCR "/bundle/2", "aXb");
// system db name
@ -2599,7 +2606,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/16384/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("8dbabb96e032b8d9f1993c0e4b9141e71ade01a1")))});
HRN_STORAGE_PUT_Z(storageRepoWrite(), TEST_REPO_PATH "base/16384/" PG_FILE_PGVERSION, PG_VERSION_94_STR "\n");
// base/16384/16385
@ -2612,7 +2619,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/16384/16385"), .size = 16384, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "d74e5f7ebe52a3ed468ba08c5b6aefaccd1ca88f"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("d74e5f7ebe52a3ed468ba08c5b6aefaccd1ca88f")))});
HRN_STORAGE_PUT(storageRepoWrite(), TEST_REPO_PATH "base/16384/16385", fileBuffer);
// base/32768 directory
@ -2628,7 +2635,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/32768/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("8dbabb96e032b8d9f1993c0e4b9141e71ade01a1")))});
HRN_STORAGE_PUT_Z(storageRepoWrite(), TEST_REPO_PATH "base/32768/" PG_FILE_PGVERSION, PG_VERSION_94_STR "\n");
// base/32768/32769
@ -2641,7 +2648,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/32768/32769"), .size = 32768, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "a40f0986acb1531ce0cc75a23dcf8aa406ae9081"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("a40f0986acb1531ce0cc75a23dcf8aa406ae9081")))});
HRN_STORAGE_PUT(storageRepoWrite(), TEST_REPO_PATH "base/32768/32769", fileBuffer);
// File link to postgresql.conf
@ -2658,7 +2665,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "postgresql.conf"), .size = 15, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "98b8abb2e681e2a5a7d8ab082c0a79727887558d"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("98b8abb2e681e2a5a7d8ab082c0a79727887558d")))});
HRN_STORAGE_PUT_Z(storageRepoWrite(), TEST_REPO_PATH "postgresql.conf", "POSTGRESQL.CONF");
// File link to pg_hba.conf
@ -2675,7 +2682,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "pg_hba.conf"), .size = 11, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "401215e092779574988a854d8c7caed7f91dba4b"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("401215e092779574988a854d8c7caed7f91dba4b")))});
HRN_STORAGE_PUT_Z(storageRepoWrite(), TEST_REPO_PATH "pg_hba.conf", "PG_HBA.CONF");
// tablespace_map (will be ignored during restore)
@ -2683,7 +2690,7 @@ testRun(void)
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = bufPtrConst(HASH_TYPE_SHA1_ZERO_BUF)});
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP);
// Path link to pg_wal

17
test/src/module/command/verifyTest.c
View File

@ -34,7 +34,7 @@ testRun(void)
const char *fileContents = "acefile";
uint64_t fileSize = 7;
const String *fileChecksum = STRDEF("d1cd8a7d11daa26814b93eb604e1d49ab4b43770");
const Buffer *fileChecksum = bufNewDecode(encodingHex, STRDEF("d1cd8a7d11daa26814b93eb604e1d49ab4b43770"));
#define TEST_BACKUP_DB1_94 \
"db-catalog-version=201409291\n" \
@ -839,7 +839,7 @@ testRun(void)
String *filePathName = strNewZ(STORAGE_REPO_ARCHIVE "/testfile");
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), strZ(filePathName));
TEST_RESULT_UINT(
verifyFile(filePathName, 0, NULL, compressTypeNone, STRDEF(HASH_TYPE_SHA1_ZERO), 0, NULL), verifyOk, "file ok");
verifyFile(filePathName, 0, NULL, compressTypeNone, HASH_TYPE_SHA1_ZERO_BUF, 0, NULL), verifyOk, "file ok");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("file size invalid in archive");
@ -870,8 +870,8 @@ testRun(void)
verifyOk, "file encrypted compressed ok");
TEST_RESULT_UINT(
verifyFile(
filePathName, 0, NULL, compressTypeGz, STRDEF("badchecksum"), fileSize, STRDEF("pass")), verifyChecksumMismatch,
"file encrypted compressed checksum mismatch");
filePathName, 0, NULL, compressTypeGz, bufNewDecode(encodingHex, STRDEF("aa")), fileSize, STRDEF("pass")),
verifyChecksumMismatch, "file encrypted compressed checksum mismatch");
}
// *****************************************************************************************************************************
@ -1155,7 +1155,7 @@ testRun(void)
TEST_MANIFEST_LINK_DEFAULT
TEST_MANIFEST_PATH
TEST_MANIFEST_PATH_DEFAULT,
strZ(fileChecksum));
strZ(strNewEncode(encodingHex, fileChecksum)));
HRN_INFO_PUT(
storageRepoIdxWrite(0), STORAGE_REPO_BACKUP "/20181119-152900F/" BACKUP_MANIFEST_FILE, strZ(manifestContent),
@ -1192,7 +1192,8 @@ testRun(void)
TEST_MANIFEST_LINK_DEFAULT
TEST_MANIFEST_PATH
TEST_MANIFEST_PATH_DEFAULT,
strZ(fileChecksum), strZ(fileChecksum), strZ(fileChecksum));
strZ(strNewEncode(encodingHex, fileChecksum)), strZ(strNewEncode(encodingHex, fileChecksum)),
strZ(strNewEncode(encodingHex, fileChecksum)));
// Write manifests for dependent backup
HRN_INFO_PUT(
@ -1471,7 +1472,7 @@ testRun(void)
TEST_MANIFEST_LINK_DEFAULT
TEST_MANIFEST_PATH
TEST_MANIFEST_PATH_DEFAULT,
strZ(fileChecksum));
strZ(strNewEncode(encodingHex, fileChecksum)));
HRN_INFO_PUT(
storageRepoWrite(), STORAGE_REPO_BACKUP "/20181119-152900F/" BACKUP_MANIFEST_FILE, strZ(manifestContent),
@ -1527,7 +1528,7 @@ testRun(void)
TEST_MANIFEST_LINK_DEFAULT
TEST_MANIFEST_PATH
TEST_MANIFEST_PATH_DEFAULT,
strZ(fileChecksum), (unsigned int)fileSize);
strZ(strNewEncode(encodingHex, fileChecksum)), (unsigned int)fileSize);
HRN_INFO_PUT(
storageRepoWrite(), STORAGE_REPO_BACKUP "/20201119-163000F/" BACKUP_MANIFEST_FILE, strZ(manifestContent),

34
test/src/module/info/manifestTest.c
View File

@ -31,7 +31,6 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_UINT(sizeof(ManifestLoadFound), TEST_64BIT() ? 1 : 1, "check size of ManifestLoadFound");
TEST_RESULT_UINT(sizeof(ManifestPath), TEST_64BIT() ? 32 : 16, "check size of ManifestPath");
TEST_RESULT_UINT(sizeof(ManifestFile), TEST_64BIT() ? 136 : 108, "check size of ManifestFile");
}
// *****************************************************************************************************************************
@ -1011,17 +1010,17 @@ testRun(void)
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.checksumSha1 = "da39a3ee5e6b4b0d3255bfef95601890afd80709"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("da39a3ee5e6b4b0d3255bfef95601890afd80709")))});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182860,
.checksumSha1 = "ccccccccccaaaaaaaaaabbbbbbbbbbdddddddddd"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("ccccccccccaaaaaaaaaabbbbbbbbbbdddddddddd")))});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd")))});
}
OBJ_NEW_END();
@ -1070,14 +1069,15 @@ testRun(void)
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-bundle"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"),
.checksumSha1 = bufPtrConst(HASH_TYPE_SHA1_ZERO_BUF)});
// Zero-length file with the copy flag which will appear to come from a non-bundled backup (so will get a reference)
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-normal"), .copy = true, .size = 0, .sizeRepo = 0,
.timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"),
.checksumSha1 = HASH_TYPE_SHA1_ZERO});
.checksumSha1 = bufPtrConst(HASH_TYPE_SHA1_ZERO_BUF)});
manifestFileAdd(
manifest,
&(ManifestFile){
@ -1089,18 +1089,19 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd")))});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-bundle"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"),
.checksumSha1 = bufPtrConst(HASH_TYPE_SHA1_ZERO_BUF)});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE0-normal"), .size = 0, .sizeRepo = 0,
.timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test"),
.checksumSha1 = HASH_TYPE_SHA1_ZERO});
.checksumSha1 = bufPtrConst(HASH_TYPE_SHA1_ZERO_BUF)});
TEST_RESULT_VOID(manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, NULL), "incremental manifest");
@ -1155,7 +1156,8 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .copy = true, .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", .checksumPage = true, .checksumPageError = true,
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"))),
.checksumPage = true, .checksumPageError = true,
.checksumPageErrorList = jsonFromVar(varNewVarLst(checksumPageErrorList))});
TEST_RESULT_VOID(manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, NULL), "incremental manifest");
@ -1210,7 +1212,7 @@ testRun(void)
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .copy = true, .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa")))});
TEST_RESULT_VOID(
manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, STRDEF("000000040000000400000004")),
@ -1277,7 +1279,7 @@ testRun(void)
manifestPrior,
&(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa")))});
TEST_RESULT_VOID(
manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, STRDEF("000000030000000300000003")), "incremental manifest");
@ -1576,7 +1578,7 @@ testRun(void)
// Munge files to produce errors
ManifestFile file = manifestFileFind(manifest, STRDEF("pg_data/postgresql.conf"));
file.checksumSha1[0] = '\0';
file.checksumSha1 = NULL;
file.sizeRepo = 0;
file.resume = true;
manifestFileUpdate(manifest, &file);
@ -1600,7 +1602,7 @@ testRun(void)
// Undo changes made to files
file = manifestFileFind(manifest, STRDEF("pg_data/postgresql.conf"));
TEST_RESULT_BOOL(file.resume, true, "resume is set");
memcpy(file.checksumSha1, "184473f470864e067ee3a22e64b47b0a1c356f29", HASH_TYPE_SHA1_SIZE_HEX + 1);
file.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("184473f470864e067ee3a22e64b47b0a1c356f29")));
file.sizeRepo = 4457;
manifestFileUpdate(manifest, &file);
@ -1768,11 +1770,11 @@ testRun(void)
// Munge the sha1 checksum to be blank
ManifestFilePack **const fileMungePack = manifestFilePackFindInternal(manifest, STRDEF("pg_data/postgresql.conf"));
ManifestFile fileMunge = manifestFileUnpack(manifest, *fileMungePack);
fileMunge.checksumSha1[0] = '\0';
fileMunge.checksumSha1 = NULL;
manifestFilePackUpdate(manifest, fileMungePack, &fileMunge);
file = manifestFileFind(manifest, STRDEF("pg_data/postgresql.conf"));
file.checksumSha1[0] = '\0';
file.checksumSha1 = NULL;
manifestFileUpdate(manifest, &file);
// ManifestDb getters