db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-03 00:26:59 +02:00
Code Issues Releases Activity

Pack manifest file structs to save memory.

Browse Source 
Manifests with a very large number of files can use a considerable amount of memory. There are a lot of zeroes in the data so it can be stored more efficiently by using base-128 varint encoding for the integers and storing the strings in the same allocation.

The downside is that the data needs to be unpacked in order to be used, but in most cases this seems fast enough (about 10% slower than before) except for saving the manifest, which is 10% slower up to 10 million files and then gets about 5x slower by 100 million (two minutes on my M1 Mac). Profiling does not show this slowdown so I wonder if this is related to the change in memory layout. Curiously, the function that increased most was jsonFromStrInternal(), which was not modified. That gives more weight to the idea that there is some kind of memory issue going on here and one hopes that servers would be less affected. Either way, they largest use cases we have seen are for about 6 million files so if we can improve that case I believe we will be better off.

Further analysis showed that most of the time was taken up writing the size and timestamp fields, which makes almost no sense. The same amount of time was used if they were hard-coded to 0, which points to some odd memory issue on the M1 architecture.

This change has been planned for a while, but the particular impetus at this time is that small file support requires additional fields that would increase manifest memory usage by about 20%, even if the feature is not used.

Note that the Pack code has been updated to use the new varint encoder, but the decoder remains separate because it needs to fetch one byte at a time.
This commit is contained in:
David Steele
2022-01-21 17:05:07 -05:00
committed by GitHub
parent 575ae77c0d
commit 61ce58692f
17 changed files with 698 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
doc/xml/release.xml
View File

@ -68,6 +68,7 @@
<commit subject="Simplify base path mode in mock/all integration tests."/> <commit subject="Simplify base path mode in mock/all integration tests."/>
<commit subject="Simplify manifest defaults."/> <commit subject="Simplify manifest defaults."/>
<commit subject="Convert varNewUInt64() to VARUINT64() where possible in manifest."/> <commit subject="Convert varNewUInt64() to VARUINT64() where possible in manifest."/>
<commit subject="Pack manifest file structs to save memory."/>
<release-item-contributor-list> <release-item-contributor-list>
<release-item-contributor id="david.steele"/> <release-item-contributor id="david.steele"/>

111
src/command/backup/backup.c
View File

@ -576,37 +576,43 @@ void backupResumeCallback(void *data, const StorageInfo *info)
if (fileCompressType != compressTypeNone) if (fileCompressType != compressTypeNone)
manifestName = compressExtStrip(manifestName, fileCompressType); manifestName = compressExtStrip(manifestName, fileCompressType);
// Find the file in both manifests
const ManifestFile *file = manifestFileFindDefault(resumeData->manifest, manifestName, NULL);
const ManifestFile *fileResume = manifestFileFindDefault(resumeData->manifestResume, manifestName, NULL);
// Check if the file can be resumed or must be removed // Check if the file can be resumed or must be removed
const char *removeReason = NULL; const char *removeReason = NULL;
if (fileCompressType != resumeData->compressType) if (fileCompressType != resumeData->compressType)
removeReason = "mismatched compression type"; removeReason = "mismatched compression type";
else if (file == NULL) else if (!manifestFileExists(resumeData->manifest, manifestName))
removeReason = "missing in manifest"; removeReason = "missing in manifest";
else if (file->reference != NULL) else
{
const ManifestFile file = manifestFileFind(resumeData->manifest, manifestName);
if (file.reference != NULL)
removeReason = "reference in manifest"; removeReason = "reference in manifest";
else if (fileResume == NULL) else if (!manifestFileExists(resumeData->manifestResume, manifestName))
removeReason = "missing in resumed manifest"; removeReason = "missing in resumed manifest";
else if (fileResume->reference != NULL) else
{
const ManifestFile fileResume = manifestFileFind(resumeData->manifestResume, manifestName);
if (fileResume.reference != NULL)
removeReason = "reference in resumed manifest"; removeReason = "reference in resumed manifest";
else if (fileResume->checksumSha1[0] == '\0') else if (fileResume.checksumSha1[0] == '\0')
removeReason = "no checksum in resumed manifest"; removeReason = "no checksum in resumed manifest";
else if (file->size != fileResume->size) else if (file.size != fileResume.size)
removeReason = "mismatched size"; removeReason = "mismatched size";
else if (!resumeData->delta && file->timestamp != fileResume->timestamp) else if (!resumeData->delta && file.timestamp != fileResume.timestamp)
removeReason = "mismatched timestamp"; removeReason = "mismatched timestamp";
else if (file->size == 0) else if (file.size == 0)
// ??? don't resume zero size files because Perl wouldn't -- this can be removed after the migration) // ??? don't resume zero size files because Perl wouldn't -- this can be removed after the migration)
removeReason = "zero size"; removeReason = "zero size";
else else
{ {
manifestFileUpdate( manifestFileUpdate(
resumeData->manifest, manifestName, file->size, fileResume->sizeRepo, fileResume->checksumSha1, NULL, resumeData->manifest, manifestName, file.size, fileResume.sizeRepo, fileResume.checksumSha1, NULL,
fileResume->checksumPage, fileResume->checksumPageError, fileResume->checksumPageErrorList); fileResume.checksumPage, fileResume.checksumPageError, fileResume.checksumPageErrorList);
}
}
} }
// Remove the file if it could not be resumed // Remove the file if it could not be resumed
@ -981,7 +987,7 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti
file.checksumSha1, strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))), file.checksumSha1, strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))),
HASH_TYPE_SHA1_SIZE_HEX + 1); HASH_TYPE_SHA1_SIZE_HEX + 1);
manifestFileAdd(manifest, &file); manifestFileAdd(manifest, file);
LOG_DETAIL_FMT("wrote '%s' file returned from pg_stop_backup()", strZ(name)); LOG_DETAIL_FMT("wrote '%s' file returned from pg_stop_backup()", strZ(name));
} }
@ -1165,7 +1171,7 @@ backupJobResult(
{ {
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
const ManifestFile *const file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job))); const ManifestFile file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
const unsigned int processId = protocolParallelJobProcessId(job); const unsigned int processId = protocolParallelJobProcessId(job);
PackRead *const jobResult = protocolParallelJobResult(job); PackRead *const jobResult = protocolParallelJobResult(job);
@ -1205,7 +1211,7 @@ backupJobResult(
else if (copyResult == backupCopyResultSkip) else if (copyResult == backupCopyResultSkip)
{ {
LOG_DETAIL_PID_FMT(processId, "skip file removed by database %s", strZ(fileLog)); LOG_DETAIL_PID_FMT(processId, "skip file removed by database %s", strZ(fileLog));
strLstAdd(fileRemove, file->name); strLstAdd(fileRemove, file.name);
} }
// Else file was copied so update manifest // Else file was copied so update manifest
else else
@ -1220,13 +1226,13 @@ backupJobResult(
" continue but this may be an issue unless the resumed backup path in the repository is known to be" " continue but this may be an issue unless the resumed backup path in the repository is known to be"
" corrupted.\n" " corrupted.\n"
"NOTE: this does not indicate a problem with the PostgreSQL page checksums.", "NOTE: this does not indicate a problem with the PostgreSQL page checksums.",
strZ(file->name), file->checksumSha1); strZ(file.name), file.checksumSha1);
} }
LOG_DETAIL_PID_FMT(processId, "backup file %s (%s)%s", strZ(fileLog), strZ(logProgress), strZ(logChecksum)); LOG_DETAIL_PID_FMT(processId, "backup file %s (%s)%s", strZ(fileLog), strZ(logProgress), strZ(logChecksum));
// If the file had page checksums calculated during the copy // If the file had page checksums calculated during the copy
ASSERT((!file->checksumPage && checksumPageResult == NULL) || (file->checksumPage && checksumPageResult != NULL)); ASSERT((!file.checksumPage && checksumPageResult == NULL) || (file.checksumPage && checksumPageResult != NULL));
bool checksumPageError = false; bool checksumPageError = false;
const VariantList *checksumPageErrorList = NULL; const VariantList *checksumPageErrorList = NULL;
@ -1300,8 +1306,8 @@ backupJobResult(
// Update file info and remove any reference to the file's existence in a prior backup // Update file info and remove any reference to the file's existence in a prior backup
manifestFileUpdate( manifestFileUpdate(
manifest, file->name, copySize, repoSize, strZ(copyChecksum), VARSTR(NULL), file->checksumPage, manifest, file.name, copySize, repoSize, strZ(copyChecksum), VARSTR(NULL), file.checksumPage,
checksumPageError, checksumPageErrorList); checksumPageError, checksumPageErrorList != NULL ? jsonFromVar(varNewVarLst(checksumPageErrorList)) : NULL);
} }
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -1420,14 +1426,18 @@ backupProcessQueueComparator(const void *item1, const void *item2)
ASSERT(item1 != NULL); ASSERT(item1 != NULL);
ASSERT(item2 != NULL); ASSERT(item2 != NULL);
// Unpack files
ManifestFile file1 = manifestFileUnpack(*(const ManifestFilePack **)item1);
ManifestFile file2 = manifestFileUnpack(*(const ManifestFilePack **)item2);
// If the size differs then that's enough to determine order // If the size differs then that's enough to determine order
if ((*(ManifestFile **)item1)->size < (*(ManifestFile **)item2)->size) if (file1.size < file2.size)
FUNCTION_TEST_RETURN(-1); FUNCTION_TEST_RETURN(-1);
else if ((*(ManifestFile **)item1)->size > (*(ManifestFile **)item2)->size) else if (file1.size > file2.size)
FUNCTION_TEST_RETURN(1); FUNCTION_TEST_RETURN(1);
// If size is the same then use name to generate a deterministic ordering (names must be unique) // If size is the same then use name to generate a deterministic ordering (names must be unique)
FUNCTION_TEST_RETURN(strCmp((*(ManifestFile **)item1)->name, (*(ManifestFile **)item2)->name)); FUNCTION_TEST_RETURN(strCmp(file1.name, file2.name));
} }
// Helper to generate the backup queues // Helper to generate the backup queues
@ -1479,20 +1489,21 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData)
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{ {
const ManifestFile *file = manifestFile(manifest, fileIdx); const ManifestFilePack *const filePack = manifestFilePackGet(manifest, fileIdx);
const ManifestFile file = manifestFileUnpack(filePack);
// If the file is a reference it should only be backed up if delta and not zero size // If the file is a reference it should only be backed up if delta and not zero size
if (file->reference != NULL && (!jobData->delta || file->size == 0)) if (file.reference != NULL && (!jobData->delta || file.size == 0))
continue; continue;
// Is pg_control in the backup? // Is pg_control in the backup?
if (strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))) if (strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)))
pgControlFound = true; pgControlFound = true;
// Files that must be copied from the primary are always put in queue 0 when backup from standby // Files that must be copied from the primary are always put in queue 0 when backup from standby
if (jobData->backupStandby && backupProcessFilePrimary(jobData->standbyExp, file->name)) if (jobData->backupStandby && backupProcessFilePrimary(jobData->standbyExp, file.name))
{ {
lstAdd(*(List **)lstGet(jobData->queueList, 0), &file); lstAdd(*(List **)lstGet(jobData->queueList, 0), &filePack);
} }
// Else find the correct queue by matching the file to a target // Else find the correct queue by matching the file to a target
else else
@ -1504,7 +1515,7 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData)
{ {
CHECK(AssertError, targetIdx < strLstSize(targetList), "backup target not found"); CHECK(AssertError, targetIdx < strLstSize(targetList), "backup target not found");
if (strBeginsWith(file->name, strLstGet(targetList, targetIdx))) if (strBeginsWith(file.name, strLstGet(targetList, targetIdx)))
break; break;
targetIdx++; targetIdx++;
@ -1512,11 +1523,11 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData)
while (1); while (1);
// Add file to queue // Add file to queue
lstAdd(*(List **)lstGet(jobData->queueList, targetIdx + queueOffset), &file); lstAdd(*(List **)lstGet(jobData->queueList, targetIdx + queueOffset), &filePack);
} }
// Add size to total // Add size to total
result += file->size; result += file.size;
// Increment total files // Increment total files
fileTotal++; fileTotal++;
@ -1600,21 +1611,21 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
if (!lstEmpty(queue)) if (!lstEmpty(queue))
{ {
const ManifestFile *file = *(ManifestFile **)lstGet(queue, 0); const ManifestFile file = manifestFileUnpack(*(ManifestFilePack **)lstGet(queue, 0));
// Create backup job // Create backup job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE);
PackWrite *const param = protocolCommandParam(command); PackWrite *const param = protocolCommandParam(command);
pckWriteStrP(param, manifestPathPg(file->name)); pckWriteStrP(param, manifestPathPg(file.name));
pckWriteBoolP(param, !strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))); pckWriteBoolP(param, !strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)));
pckWriteU64P(param, file->size); pckWriteU64P(param, file.size);
pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file->name)); pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file.name));
pckWriteStrP(param, file->checksumSha1[0] != 0 ? STR(file->checksumSha1) : NULL); pckWriteStrP(param, file.checksumSha1[0] != 0 ? STR(file.checksumSha1) : NULL);
pckWriteBoolP(param, file->checksumPage); pckWriteBoolP(param, file.checksumPage);
pckWriteU64P(param, jobData->lsnStart); pckWriteU64P(param, jobData->lsnStart);
pckWriteStrP(param, file->name); pckWriteStrP(param, file.name);
pckWriteBoolP(param, file->reference != NULL); pckWriteBoolP(param, file.reference != NULL);
pckWriteU32P(param, jobData->compressType); pckWriteU32P(param, jobData->compressType);
pckWriteI32P(param, jobData->compressLevel); pckWriteI32P(param, jobData->compressLevel);
pckWriteStrP(param, jobData->backupLabel); pckWriteStrP(param, jobData->backupLabel);
@ -1628,7 +1639,7 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
// Assign job to result // Assign job to result
MEM_CONTEXT_PRIOR_BEGIN() MEM_CONTEXT_PRIOR_BEGIN()
{ {
result = protocolParallelJobNew(VARSTR(file->name), command); result = protocolParallelJobNew(VARSTR(file.name), command);
} }
MEM_CONTEXT_PRIOR_END(); MEM_CONTEXT_PRIOR_END();
@ -1774,7 +1785,7 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart
backupStandby && protocolParallelJobProcessId(job) > 1 ? backupData->hostStandby : backupData->hostPrimary, backupStandby && protocolParallelJobProcessId(job) > 1 ? backupData->hostStandby : backupData->hostPrimary,
storagePathP( storagePathP(
protocolParallelJobProcessId(job) > 1 ? storagePgIdx(pgIdx) : backupData->storagePrimary, protocolParallelJobProcessId(job) > 1 ? storagePgIdx(pgIdx) : backupData->storagePrimary,
manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job)))->name)), fileRemove, job, manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job))).name)), fileRemove, job,
sizeTotal, &sizeProgress); sizeTotal, &sizeProgress);
} }
@ -1814,22 +1825,22 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{ {
const ManifestFile *const file = manifestFile(manifest, fileIdx); const ManifestFile file = manifestFile(manifest, fileIdx);
// If the file has a reference, then it was not copied since it can be retrieved from the referenced backup. However, // If the file has a reference, then it was not copied since it can be retrieved from the referenced backup. However,
// if hardlinking is enabled the link will need to be created. // if hardlinking is enabled the link will need to be created.
if (file->reference != NULL) if (file.reference != NULL)
{ {
// If hardlinking is enabled then create a hardlink for files that have not changed since the last backup // If hardlinking is enabled then create a hardlink for files that have not changed since the last backup
if (hardLink) if (hardLink)
{ {
LOG_DETAIL_FMT("hardlink %s to %s", strZ(file->name), strZ(file->reference)); LOG_DETAIL_FMT("hardlink %s to %s", strZ(file.name), strZ(file.reference));
const String *const linkName = storagePathP( const String *const linkName = storagePathP(
storageRepo(), strNewFmt("%s/%s%s", strZ(backupPathExp), strZ(file->name), compressExt)); storageRepo(), strNewFmt("%s/%s%s", strZ(backupPathExp), strZ(file.name), compressExt));
const String *const linkDestination = storagePathP( const String *const linkDestination = storagePathP(
storageRepo(), storageRepo(),
strNewFmt(STORAGE_REPO_BACKUP "/%s/%s%s", strZ(file->reference), strZ(file->name), compressExt)); strNewFmt(STORAGE_REPO_BACKUP "/%s/%s%s", strZ(file.reference), strZ(file.name), compressExt));
THROW_ON_SYS_ERROR_FMT( THROW_ON_SYS_ERROR_FMT(
link(strZ(linkDestination), strZ(linkName)) == -1, FileOpenError, link(strZ(linkDestination), strZ(linkName)) == -1, FileOpenError,
@ -1838,7 +1849,7 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart
// Else log the reference. With delta, it is possible that references may have been removed if a file needed to be // Else log the reference. With delta, it is possible that references may have been removed if a file needed to be
// recopied. // recopied.
else else
LOG_DETAIL_FMT("reference %s to %s", strZ(file->name), strZ(file->reference)); LOG_DETAIL_FMT("reference %s to %s", strZ(file.name), strZ(file.reference));
} }
} }
@ -1974,7 +1985,7 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif
memcpy(file.checksumSha1, strZ(strSubN(archiveFile, 25, 40)), HASH_TYPE_SHA1_SIZE_HEX + 1); memcpy(file.checksumSha1, strZ(strSubN(archiveFile, 25, 40)), HASH_TYPE_SHA1_SIZE_HEX + 1);
manifestFileAdd(manifest, &file); manifestFileAdd(manifest, file);
} }
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();

6
src/command/info/info.c
View File

@ -533,10 +533,10 @@ backupListAdd(
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(repoData->manifest); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(repoData->manifest); fileIdx++)
{ {
const ManifestFile *file = manifestFile(repoData->manifest, fileIdx); const ManifestFile file = manifestFile(repoData->manifest, fileIdx);
if (file->checksumPageError) if (file.checksumPageError)
varLstAdd(checksumPageErrorList, varNewStr(manifestPathPg(file->name))); varLstAdd(checksumPageErrorList, varNewStr(manifestPathPg(file.name)));
} }
if (!varLstEmpty(checksumPageErrorList)) if (!varLstEmpty(checksumPageErrorList))

112
src/command/restore/restore.c
View File

@ -533,22 +533,22 @@ restoreManifestMap(Manifest *manifest)
if (target.name == NULL) if (target.name == NULL)
{ {
// Is the specified link a file or a path? Error if they both match. // Is the specified link a file or a path? Error if they both match.
const ManifestPath *const path = manifestPathFindDefault(manifest, manifestName, NULL); const bool pathExists = manifestPathFindDefault(manifest, manifestName, NULL) != NULL;
const ManifestFile *const file = manifestFileFindDefault(manifest, manifestName, NULL); const bool fileExists = manifestFileExists(manifest, manifestName);
CHECK(FormatError, path == NULL || file == NULL, "link may not be both file and path"); CHECK(FormatError, !pathExists || !fileExists, "link may not be both file and path");
target = (ManifestTarget){.name = manifestName, .path = linkPath, .type = manifestTargetTypeLink}; target = (ManifestTarget){.name = manifestName, .path = linkPath, .type = manifestTargetTypeLink};
// If a file // If a file
if (file != NULL) if (fileExists)
{ {
// File needs to be set so the file/path is updated later but set it to something invalid just in case it // File needs to be set so the file/path is updated later but set it to something invalid just in case it
// it does not get updated due to a regression // it does not get updated due to a regression
target.file = DOT_STR; target.file = DOT_STR;
} }
// Else error if not a path // Else error if not a path
else if (path == NULL) else if (!pathExists)
{ {
THROW_FMT( THROW_FMT(
LinkMapError, LinkMapError,
@ -666,20 +666,20 @@ restoreManifestOwnerReplace(const String *const owner, const String *const owner
} }
// Helper to get list of owners from a file/link/path list // Helper to get list of owners from a file/link/path list
#define RESTORE_MANIFEST_OWNER_GET(type) \ #define RESTORE_MANIFEST_OWNER_GET(type, deref) \
for (unsigned int itemIdx = 0; itemIdx < manifest##type##Total(manifest); itemIdx++) \ for (unsigned int itemIdx = 0; itemIdx < manifest##type##Total(manifest); itemIdx++) \
{ \ { \
Manifest##type *item = (Manifest##type *)manifest##type(manifest, itemIdx); \ Manifest##type item = deref manifest##type(manifest, itemIdx); \
\ \
if (item->user == NULL) \ if (item.user == NULL) \
userNull = true; \ userNull = true; \
else \ else \
strLstAddIfMissing(userList, item->user); \ strLstAddIfMissing(userList, item.user); \
\ \
if (item->group == NULL) \ if (item.group == NULL) \
groupNull = true; \ groupNull = true; \
else \ else \
strLstAddIfMissing(groupList, item->group); \ strLstAddIfMissing(groupList, item.group); \
} }
// Helper to warn when an owner is missing and must be remapped // Helper to warn when an owner is missing and must be remapped
@ -719,9 +719,9 @@ restoreManifestOwner(const Manifest *const manifest, const String **const rootRe
bool groupNull = false; bool groupNull = false;
StringList *groupList = strLstNew(); StringList *groupList = strLstNew();
RESTORE_MANIFEST_OWNER_GET(File); RESTORE_MANIFEST_OWNER_GET(File, );
RESTORE_MANIFEST_OWNER_GET(Link); RESTORE_MANIFEST_OWNER_GET(Link, *(ManifestLink *));
RESTORE_MANIFEST_OWNER_GET(Path); RESTORE_MANIFEST_OWNER_GET(Path, *(ManifestPath *));
// Update users and groups in the manifest (this can only be done as root) // Update users and groups in the manifest (this can only be done as root)
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
@ -918,14 +918,15 @@ restoreCleanInfoListCallback(void *data, const StorageInfo *info)
{ {
case storageTypeFile: case storageTypeFile:
{ {
const ManifestFile *manifestFile = manifestFileFindDefault(cleanData->manifest, manifestName, NULL); if (manifestFileExists(cleanData->manifest, manifestName) &&
manifestLinkFindDefault(cleanData->manifest, manifestName, NULL) == NULL)
if (manifestFile != NULL && manifestLinkFindDefault(cleanData->manifest, manifestName, NULL) == NULL)
{ {
const ManifestFile manifestFile = manifestFileFind(cleanData->manifest, manifestName);
restoreCleanOwnership( restoreCleanOwnership(
pgPath, manifestFile->user, cleanData->rootReplaceUser, manifestFile->group, cleanData->rootReplaceGroup, pgPath, manifestFile.user, cleanData->rootReplaceUser, manifestFile.group, cleanData->rootReplaceGroup,
info->userId, info->groupId, false); info->userId, info->groupId, false);
restoreCleanMode(pgPath, manifestFile->mode, info); restoreCleanMode(pgPath, manifestFile.mode, info);
} }
else else
{ {
@ -1133,7 +1134,7 @@ restoreCleanBuild(const Manifest *const manifest, const String *const rootReplac
// Skip the tablespace_map file when present so PostgreSQL does not rewrite links in pg_tblspc. The tablespace links will be // Skip the tablespace_map file when present so PostgreSQL does not rewrite links in pg_tblspc. The tablespace links will be
// created after paths are cleaned. // created after paths are cleaned.
if (manifestFileFindDefault(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_TABLESPACEMAP), NULL) != NULL && if (manifestFileExists(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_TABLESPACEMAP)) &&
manifestData(manifest)->pgVersion >= PG_VERSION_TABLESPACE_MAP) manifestData(manifest)->pgVersion >= PG_VERSION_TABLESPACE_MAP)
{ {
LOG_DETAIL_FMT("skip '" PG_FILE_TABLESPACEMAP "' -- tablespace links will be created based on mappings"); LOG_DETAIL_FMT("skip '" PG_FILE_TABLESPACEMAP "' -- tablespace links will be created based on mappings");
@ -1142,7 +1143,7 @@ restoreCleanBuild(const Manifest *const manifest, const String *const rootReplac
// Skip postgresql.auto.conf if preserve is set and the PostgreSQL version supports recovery GUCs // Skip postgresql.auto.conf if preserve is set and the PostgreSQL version supports recovery GUCs
if (manifestData(manifest)->pgVersion >= PG_VERSION_RECOVERY_GUC && cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_PRESERVE && if (manifestData(manifest)->pgVersion >= PG_VERSION_RECOVERY_GUC && cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_PRESERVE &&
manifestFileFindDefault(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF), NULL) != NULL) manifestFileExists(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)))
{ {
LOG_DETAIL_FMT("skip '" PG_FILE_POSTGRESQLAUTOCONF "' -- recovery type is preserve"); LOG_DETAIL_FMT("skip '" PG_FILE_POSTGRESQLAUTOCONF "' -- recovery type is preserve");
manifestFileRemove(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)); manifestFileRemove(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF));
@ -1341,11 +1342,11 @@ restoreSelectiveExpression(Manifest *manifest)
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{ {
const ManifestFile *file = manifestFile(manifest, fileIdx); const String *const fileName = manifestFileNameGet(manifest, fileIdx);
if (regExpMatch(baseRegExp, file->name) || regExpMatch(tablespaceRegExp, file->name)) if (regExpMatch(baseRegExp, fileName) || regExpMatch(tablespaceRegExp, fileName))
{ {
String *dbId = strBase(strPath(file->name)); String *dbId = strBase(strPath(fileName));
// In the highly unlikely event that a system database was somehow added after the backup began, it will only be // In the highly unlikely event that a system database was somehow added after the backup began, it will only be
// found in the file list and not the manifest db section, so add it to the system database list // found in the file list and not the manifest db section, so add it to the system database list
@ -1907,14 +1908,18 @@ restoreProcessQueueComparator(const void *item1, const void *item2)
ASSERT(item1 != NULL); ASSERT(item1 != NULL);
ASSERT(item2 != NULL); ASSERT(item2 != NULL);
// Unpack files
ManifestFile file1 = manifestFileUnpack(*(const ManifestFilePack **)item1);
ManifestFile file2 = manifestFileUnpack(*(const ManifestFilePack **)item2);
// If the size differs then that's enough to determine order // If the size differs then that's enough to determine order
if ((*(ManifestFile **)item1)->size < (*(ManifestFile **)item2)->size) if (file1.size < file2.size)
FUNCTION_TEST_RETURN(-1); FUNCTION_TEST_RETURN(-1);
else if ((*(ManifestFile **)item1)->size > (*(ManifestFile **)item2)->size) else if (file1.size > file2.size)
FUNCTION_TEST_RETURN(1); FUNCTION_TEST_RETURN(1);
// If size is the same then use name to generate a deterministic ordering (names must be unique) // If size is the same then use name to generate a deterministic ordering (names must be unique)
FUNCTION_TEST_RETURN(strCmp((*(ManifestFile **)item1)->name, (*(ManifestFile **)item2)->name)); FUNCTION_TEST_RETURN(strCmp(file1.name, file2.name));
} }
static uint64_t static uint64_t
@ -1960,7 +1965,8 @@ restoreProcessQueue(Manifest *manifest, List **queueList)
// Now put all files into the processing queues // Now put all files into the processing queues
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{ {
const ManifestFile *file = manifestFile(manifest, fileIdx); const ManifestFilePack *const filePack = manifestFilePackGet(manifest, fileIdx);
const ManifestFile file = manifestFileUnpack(filePack);
// Find the target that contains this file // Find the target that contains this file
unsigned int targetIdx = 0; unsigned int targetIdx = 0;
@ -1970,7 +1976,7 @@ restoreProcessQueue(Manifest *manifest, List **queueList)
// A target should always be found // A target should always be found
CHECK(FormatError, targetIdx < strLstSize(targetList), "backup target not found"); CHECK(FormatError, targetIdx < strLstSize(targetList), "backup target not found");
if (strBeginsWith(file->name, strLstGet(targetList, targetIdx))) if (strBeginsWith(file.name, strLstGet(targetList, targetIdx)))
break; break;
targetIdx++; targetIdx++;
@ -1978,10 +1984,10 @@ restoreProcessQueue(Manifest *manifest, List **queueList)
while (1); while (1);
// Add file to queue // Add file to queue
lstAdd(*(List **)lstGet(*queueList, targetIdx), &file); lstAdd(*(List **)lstGet(*queueList, targetIdx), &filePack);
// Add size to total // Add size to total
result += file->size; result += file.size;
} }
// Sort the queues // Sort the queues
@ -2052,8 +2058,8 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
{ {
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
const ManifestFile *file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job))); const ManifestFile file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
bool zeroed = restoreFileZeroed(file->name, zeroExp); bool zeroed = restoreFileZeroed(file.name, zeroExp);
bool copy = pckReadBoolP(protocolParallelJobResult(job)); bool copy = pckReadBoolP(protocolParallelJobResult(job));
String *log = strCatZ(strNew(), "restore"); String *log = strCatZ(strNew(), "restore");
@ -2063,7 +2069,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
strCatZ(log, " zeroed"); strCatZ(log, " zeroed");
// Add filename // Add filename
strCatFmt(log, " file %s", strZ(restoreFilePgPath(manifest, file->name))); strCatFmt(log, " file %s", strZ(restoreFilePgPath(manifest, file.name)));
// If not copied and not zeroed add details to explain why it was not copied // If not copied and not zeroed add details to explain why it was not copied
if (!copy && !zeroed) if (!copy && !zeroed)
@ -2074,8 +2080,8 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
if (cfgOptionBool(cfgOptForce)) if (cfgOptionBool(cfgOptForce))
{ {
strCatFmt( strCatFmt(
log, "exists and matches size %" PRIu64 " and modification time %" PRIu64, file->size, log, "exists and matches size %" PRIu64 " and modification time %" PRIu64, file.size,
(uint64_t)file->timestamp); (uint64_t)file.timestamp);
} }
// Else a checksum delta or file is zero-length // Else a checksum delta or file is zero-length
else else
@ -2083,7 +2089,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
strCatZ(log, "exists and "); strCatZ(log, "exists and ");
// No need to copy zero-length files // No need to copy zero-length files
if (file->size == 0) if (file.size == 0)
{ {
strCatZ(log, "is zero size"); strCatZ(log, "is zero size");
} }
@ -2094,12 +2100,12 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
} }
// Add size and percent complete // Add size and percent complete
sizeRestored += file->size; sizeRestored += file.size;
strCatFmt(log, " (%s, %" PRIu64 "%%)", strZ(strSizeFormat(file->size)), sizeRestored * 100 / sizeTotal); strCatFmt(log, " (%s, %" PRIu64 "%%)", strZ(strSizeFormat(file.size)), sizeRestored * 100 / sizeTotal);
// If not zero-length add the checksum // If not zero-length add the checksum
if (file->size != 0 && !zeroed) if (file.size != 0 && !zeroed)
strCatFmt(log, " checksum %s", file->checksumSha1); strCatFmt(log, " checksum %s", file.checksumSha1);
LOG_DETAIL_PID(protocolParallelJobProcessId(job), strZ(log)); LOG_DETAIL_PID(protocolParallelJobProcessId(job), strZ(log));
} }
@ -2178,24 +2184,24 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
if (!lstEmpty(queue)) if (!lstEmpty(queue))
{ {
const ManifestFile *file = *(ManifestFile **)lstGet(queue, 0); const ManifestFile file = manifestFileUnpack(*(ManifestFilePack **)lstGet(queue, 0));
// Create restore job // Create restore job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
PackWrite *const param = protocolCommandParam(command); PackWrite *const param = protocolCommandParam(command);
pckWriteStrP(param, file->name); pckWriteStrP(param, file.name);
pckWriteU32P(param, jobData->repoIdx); pckWriteU32P(param, jobData->repoIdx);
pckWriteStrP(param, file->reference != NULL ? file->reference : manifestData(jobData->manifest)->backupLabel); pckWriteStrP(param, file.reference != NULL ? file.reference : manifestData(jobData->manifest)->backupLabel);
pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType); pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType);
pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file->name)); pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file.name));
pckWriteStrP(param, STR(file->checksumSha1)); pckWriteStrP(param, STR(file.checksumSha1));
pckWriteBoolP(param, restoreFileZeroed(file->name, jobData->zeroExp)); pckWriteBoolP(param, restoreFileZeroed(file.name, jobData->zeroExp));
pckWriteU64P(param, file->size); pckWriteU64P(param, file.size);
pckWriteTimeP(param, file->timestamp); pckWriteTimeP(param, file.timestamp);
pckWriteModeP(param, file->mode); pckWriteModeP(param, file.mode);
pckWriteStrP(param, restoreManifestOwnerReplace(file->user, jobData->rootReplaceUser)); pckWriteStrP(param, restoreManifestOwnerReplace(file.user, jobData->rootReplaceUser));
pckWriteStrP(param, restoreManifestOwnerReplace(file->group, jobData->rootReplaceGroup)); pckWriteStrP(param, restoreManifestOwnerReplace(file.group, jobData->rootReplaceGroup));
pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart); pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart);
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta)); pckWriteBoolP(param, cfgOptionBool(cfgOptDelta));
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce)); pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce));
@ -2207,7 +2213,7 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
// Assign job to result // Assign job to result
MEM_CONTEXT_PRIOR_BEGIN() MEM_CONTEXT_PRIOR_BEGIN()
{ {
result = protocolParallelJobNew(VARSTR(file->name), command); result = protocolParallelJobNew(VARSTR(file.name), command);
} }
MEM_CONTEXT_PRIOR_END(); MEM_CONTEXT_PRIOR_END();

18
src/command/verify/verify.c
View File

@ -902,7 +902,7 @@ verifyBackup(void *data)
{ {
do do
{ {
const ManifestFile *fileData = manifestFile(jobData->manifest, jobData->manifestFileIdx); const ManifestFile fileData = manifestFile(jobData->manifest, jobData->manifestFileIdx);
String *filePathName = NULL; String *filePathName = NULL;
@ -910,16 +910,16 @@ verifyBackup(void *data)
backupResult->totalFileVerify++; backupResult->totalFileVerify++;
// Check if the file is referenced in a prior backup // Check if the file is referenced in a prior backup
if (fileData->reference != NULL) if (fileData.reference != NULL)
{ {
// If the prior backup is not in the result list, then that backup was never processed (likely due to the --set // If the prior backup is not in the result list, then that backup was never processed (likely due to the --set
// option) so verify the file // option) so verify the file
unsigned int backupPriorIdx = lstFindIdx(jobData->backupResultList, &fileData->reference); unsigned int backupPriorIdx = lstFindIdx(jobData->backupResultList, &fileData.reference);
if (backupPriorIdx == LIST_NOT_FOUND) if (backupPriorIdx == LIST_NOT_FOUND)
{ {
filePathName = strNewFmt( filePathName = strNewFmt(
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name), STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData.reference), strZ(fileData.name),
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
} }
// Else the backup this file references has a result so check the processing state for the referenced backup // Else the backup this file references has a result so check the processing state for the referenced backup
@ -931,14 +931,14 @@ verifyBackup(void *data)
if (!backupResultPrior->fileVerifyComplete) if (!backupResultPrior->fileVerifyComplete)
{ {
filePathName = strNewFmt( filePathName = strNewFmt(
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name), STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData.reference), strZ(fileData.name),
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
} }
// Else skip verification // Else skip verification
else else
{ {
String *priorFile = strNewFmt( String *priorFile = strNewFmt(
"%s/%s%s", strZ(fileData->reference), strZ(fileData->name), "%s/%s%s", strZ(fileData.reference), strZ(fileData.name),
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
unsigned int backupPriorInvalidIdx = lstFindIdx(backupResultPrior->invalidFileList, &priorFile); unsigned int backupPriorInvalidIdx = lstFindIdx(backupResultPrior->invalidFileList, &priorFile);
@ -965,7 +965,7 @@ verifyBackup(void *data)
else else
{ {
filePathName = strNewFmt( filePathName = strNewFmt(
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(backupResult->backupLabel), strZ(fileData->name), STORAGE_REPO_BACKUP "/%s/%s%s", strZ(backupResult->backupLabel), strZ(fileData.name),
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
} }
@ -978,8 +978,8 @@ verifyBackup(void *data)
pckWriteStrP(param, filePathName); pckWriteStrP(param, filePathName);
// If the checksum is not present in the manifest, it will be calculated by manifest load // If the checksum is not present in the manifest, it will be calculated by manifest load
pckWriteStrP(param, STR(fileData->checksumSha1)); pckWriteStrP(param, STR(fileData.checksumSha1));
pckWriteU64P(param, fileData->size); pckWriteU64P(param, fileData.size);
pckWriteStrP(param, jobData->backupCipherPass); pckWriteStrP(param, jobData->backupCipherPass);
// Assign job to result (prepend backup label being processed to the key since some files are in a prior backup) // Assign job to result (prepend backup label being processed to the key since some files are in a prior backup)

77
src/common/type/convert.c
View File

@ -482,3 +482,80 @@ cvtZToUInt64(const char *value)
FUNCTION_TEST_RETURN(cvtZToUInt64Base(value, 10)); FUNCTION_TEST_RETURN(cvtZToUInt64Base(value, 10));
} }
/**********************************************************************************************************************************/
void
cvtUInt64ToVarInt128(uint64_t value, uint8_t *const buffer, size_t *const bufferPos, const size_t bufferSize)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(UINT64, value);
FUNCTION_TEST_PARAM_P(VOID, buffer);
FUNCTION_TEST_PARAM(UINT64, bufferSize);
FUNCTION_TEST_END();
ASSERT(buffer != NULL);
ASSERT(bufferPos != NULL);
ASSERT(bufferSize > *bufferPos);
// Keep encoding bytes while the remaining value is greater than 7 bits
while (value >= 0x80)
{
// Encode the lower order 7 bits, adding the continuation bit to indicate there is more data
buffer[*bufferPos] = (unsigned char)value | 0x80;
// Shift the value to remove bits that have been encoded
value >>= 7;
// Keep track of size so we know how many bytes to write out
(*bufferPos)++;
// Make sure the buffer won't overflow
if (*bufferPos >= bufferSize)
THROW(AssertError, "buffer overflow");
}
// Encode the last 7 bits of value
buffer[*bufferPos] = (unsigned char)value;
(*bufferPos)++;
FUNCTION_TEST_RETURN_VOID();
}
uint64_t
cvtUInt64FromVarInt128(const uint8_t *const value, size_t *const valuePos)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, value);
FUNCTION_TEST_PARAM_P(UINT64, valuePos);
FUNCTION_TEST_END();
ASSERT(value != NULL);
ASSERT(valuePos != NULL);
// Decode all bytes
uint64_t result = 0;
uint8_t byte;
for (unsigned int valueIdx = 0; valueIdx < CVT_VARINT128_BUFFER_SIZE; valueIdx++)
{
// Get the next encoded byte
byte = value[*valuePos];
// Shift the lower order 7 encoded bits into the uint64 in reverse order
result |= (uint64_t)(byte & 0x7f) << (7 * valueIdx);
// Increment value position to indicate that the byte has been processed
(*valuePos)++;
// Done if the high order bit is not set to indicate more data
if (byte < 0x80)
break;
}
// By this point all bytes should have been read so error if this is not the case. This could be due to a coding error or
// corrupton in the data stream.
if (byte >= 0x80)
THROW(FormatError, "unterminated varint-128 integer");
FUNCTION_TEST_RETURN(result);
}

5
src/common/type/convert.h
View File

@ -15,6 +15,7 @@ Required buffer sizes
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
#define CVT_BOOL_BUFFER_SIZE 6 #define CVT_BOOL_BUFFER_SIZE 6
#define CVT_BASE10_BUFFER_SIZE 64 #define CVT_BASE10_BUFFER_SIZE 64
#define CVT_VARINT128_BUFFER_SIZE 10
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
@ -85,6 +86,10 @@ size_t cvtUInt64ToZ(uint64_t value, char *buffer, size_t bufferSize);
uint64_t cvtZToUInt64(const char *value); uint64_t cvtZToUInt64(const char *value);
uint64_t cvtZToUInt64Base(const char *value, int base); uint64_t cvtZToUInt64Base(const char *value, int base);
// Convert uint64 to base-128 varint and vice versa
void cvtUInt64ToVarInt128(uint64_t value, uint8_t *buffer, size_t *bufferPos, size_t bufferSize);
uint64_t cvtUInt64FromVarInt128(const uint8_t *value, size_t *valuePos);
// Convert boolean to zero-terminated string. Use cvtBoolToConstZ() whenever possible since it is more efficient. // Convert boolean to zero-terminated string. Use cvtBoolToConstZ() whenever possible since it is more efficient.
size_t cvtBoolToZ(bool value, char *buffer, size_t bufferSize); size_t cvtBoolToZ(bool value, char *buffer, size_t bufferSize);
const char *cvtBoolToConstZ(bool value); const char *cvtBoolToConstZ(bool value);

29
src/common/type/pack.c
View File

@ -104,11 +104,6 @@ Array and object types:
#include "common/type/convert.h" #include "common/type/convert.h"
#include "common/type/pack.h" #include "common/type/pack.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define PACK_UINT64_SIZE_MAX 10
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Map PackType types to the types that will be written into the pack. This hides the details of the type IDs from the user and allows Map PackType types to the types that will be written into the pack. This hides the details of the type IDs from the user and allows
the IDs used in the pack to differ from the IDs the user sees. the IDs used in the pack to differ from the IDs the user sees.
@ -495,7 +490,7 @@ pckReadU64Internal(PackRead *this)
uint8_t byte; uint8_t byte;
// Convert bytes from varint-128 encoding to a uint64 // Convert bytes from varint-128 encoding to a uint64
for (unsigned int bufferIdx = 0; bufferIdx < PACK_UINT64_SIZE_MAX; bufferIdx++) for (unsigned int bufferIdx = 0; bufferIdx < CVT_VARINT128_BUFFER_SIZE; bufferIdx++)
{ {
// Get the next encoded byte // Get the next encoded byte
pckReadBuffer(this, 1); pckReadBuffer(this, 1);
@ -1409,7 +1404,7 @@ pckWriteBuffer(PackWrite *this, const Buffer *buffer)
Pack an unsigned 64-bit integer to base-128 varint encoding Pack an unsigned 64-bit integer to base-128 varint encoding
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
static void static void
pckWriteU64Internal(PackWrite *this, uint64_t value) pckWriteU64Internal(PackWrite *const this, const uint64_t value)
{ {
FUNCTION_TEST_BEGIN(); FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PACK_WRITE, this); FUNCTION_TEST_PARAM(PACK_WRITE, this);
@ -1418,27 +1413,13 @@ pckWriteU64Internal(PackWrite *this, uint64_t value)
ASSERT(this != NULL); ASSERT(this != NULL);
unsigned char buffer[PACK_UINT64_SIZE_MAX]; unsigned char buffer[CVT_VARINT128_BUFFER_SIZE];
size_t size = 0; size_t size = 0;
// Convert uint64 to varint-128 encoding. Keep writing out bytes while the remaining value is greater than 7 bits. cvtUInt64ToVarInt128(value, buffer, &size, sizeof(buffer));
while (value >= 0x80)
{
// Encode the lower order 7 bits, adding the continuation bit to indicate there is more data
buffer[size] = (unsigned char)value | 0x80;
// Shift the value to remove bits that have been encoded
value >>= 7;
// Keep track of size so we know how many bytes to write out
size++;
}
// Encode the last 7 bits of value
buffer[size] = (unsigned char)value;
// Write encoded bytes to the buffer // Write encoded bytes to the buffer
pckWriteBuffer(this, BUF(buffer, size + 1)); pckWriteBuffer(this, BUF(buffer, size));
FUNCTION_TEST_RETURN_VOID(); FUNCTION_TEST_RETURN_VOID();
} }

16
src/info/infoBackup.c
View File

@ -370,22 +370,22 @@ infoBackupDataAdd(const InfoBackup *this, const Manifest *manifest)
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{ {
const ManifestFile *file = manifestFile(manifest, fileIdx); const ManifestFile file = manifestFile(manifest, fileIdx);
backupSize += file->size; backupSize += file.size;
backupRepoSize += file->sizeRepo > 0 ? file->sizeRepo : file->size; backupRepoSize += file.sizeRepo > 0 ? file.sizeRepo : file.size;
// If a reference to a file exists, then it is in a previous backup and the delta calculation was already done // If a reference to a file exists, then it is in a previous backup and the delta calculation was already done
if (file->reference != NULL) if (file.reference != NULL)
strLstAddIfMissing(referenceList, file->reference); strLstAddIfMissing(referenceList, file.reference);
else else
{ {
backupSizeDelta += file->size; backupSizeDelta += file.size;
backupRepoSizeDelta += file->sizeRepo > 0 ? file->sizeRepo : file->size; backupRepoSizeDelta += file.sizeRepo > 0 ? file.sizeRepo : file.size;
} }
// Is there an error in the file? // Is there an error in the file?
if (file->checksumPageError) if (file.checksumPageError)
backupError = true; backupError = true;
} }

410
src/info/manifest.c
View File

@ -198,59 +198,208 @@ manifestDbAdd(Manifest *this, const ManifestDb *db)
FUNCTION_TEST_RETURN_VOID(); FUNCTION_TEST_RETURN_VOID();
} }
// Base time used as a delta to reduce the size of packed timestamps. This will be set on the first call to manifestFilePack().
static time_t manifestPackBaseTime = -1;
// Flags used to reduce the size of packed data. They should be ordered from most to least likely and can be reordered at will.
typedef enum
{
manifestFilePackFlagReference,
manifestFilePackFlagChecksumPage,
manifestFilePackFlagChecksumPageError,
manifestFilePackFlagChecksumPageErrorList,
} ManifestFilePackFlag;
// Pack file into a compact format to save memory
static ManifestFilePack *
manifestFilePack(const ManifestFile *const file)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST_FILE, file);
FUNCTION_TEST_END();
uint8_t buffer[512];
size_t bufferPos = 0;
// Flags
uint64_t flag = 0;
if (file->checksumPage)
flag |= 1 << manifestFilePackFlagChecksumPage;
if (file->checksumPageError)
flag |= 1 << manifestFilePackFlagChecksumPageError;
if (file->checksumPageErrorList != NULL)
flag |= 1 << manifestFilePackFlagChecksumPageErrorList;
if (file->reference != NULL)
flag |= 1 << manifestFilePackFlagReference;
cvtUInt64ToVarInt128(flag, buffer, &bufferPos, sizeof(buffer));
// Size
cvtUInt64ToVarInt128(file->size, buffer, &bufferPos, sizeof(buffer));
// Use the first timestamp that appears as the base for all other timestamps. Ideally we would like a timestamp as close to the
// middle as possible but it doesn't seem worth doing the calculation.
if (manifestPackBaseTime == -1)
manifestPackBaseTime = file->timestamp;
// Timestamp
cvtUInt64ToVarInt128(cvtInt64ToZigZag(manifestPackBaseTime - file->timestamp), buffer, &bufferPos, sizeof(buffer));
// SHA1 checksum
strcpy((char *)buffer + bufferPos, file->checksumSha1);
bufferPos += HASH_TYPE_SHA1_SIZE_HEX + 1;
// Reference
if (file->reference != NULL)
cvtUInt64ToVarInt128((uintptr_t)file->reference, buffer, &bufferPos, sizeof(buffer));
// Mode
cvtUInt64ToVarInt128(file->mode, buffer, &bufferPos, sizeof(buffer));
// User/group
cvtUInt64ToVarInt128((uintptr_t)file->user, buffer, &bufferPos, sizeof(buffer));
cvtUInt64ToVarInt128((uintptr_t)file->group, buffer, &bufferPos, sizeof(buffer));
// Repo size
cvtUInt64ToVarInt128(file->sizeRepo, buffer, &bufferPos, sizeof(buffer));
// Allocate memory for the file pack
uint8_t *const result = memNew(
sizeof(StringPub) + strSize(file->name) + 1 + bufferPos + (file->checksumPageErrorList != NULL ?
sizeof(StringPub) + strSize(file->checksumPageErrorList) + 1 : 0));
// Create string object for the file name
*(StringPub *)result = (StringPub){.size = (unsigned int)strSize(file->name), .buffer = (char *)result + sizeof(StringPub)};
size_t resultPos = sizeof(StringPub);
memcpy(result + resultPos, (uint8_t *)strZ(file->name), strSize(file->name) + 1);
resultPos += strSize(file->name) + 1;
// Copy pack data
memcpy(result + resultPos, buffer, bufferPos);
// Create string object for the checksum error list
if (file->checksumPageErrorList != NULL)
{
resultPos += bufferPos;
*(StringPub *)(result + resultPos) = (StringPub)
{.size = (unsigned int)strSize(file->checksumPageErrorList), .buffer = (char *)result + resultPos + sizeof(StringPub)};
resultPos += sizeof(StringPub);
memcpy(result + resultPos, (uint8_t *)strZ(file->checksumPageErrorList), strSize(file->checksumPageErrorList) + 1);
}
FUNCTION_TEST_RETURN((ManifestFilePack *)result);
}
ManifestFile
manifestFileUnpack(const ManifestFilePack *const filePack)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, filePack);
FUNCTION_TEST_END();
ASSERT(filePack != NULL);
ASSERT(manifestPackBaseTime != -1);
ManifestFile result = {0};
size_t bufferPos = 0;
// Name
result.name = (const String *)filePack;
bufferPos += sizeof(StringPub) + strSize(result.name) + 1;
// Flags
const uint64_t flag = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// Size
result.size = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// Timestamp
result.timestamp =
manifestPackBaseTime - (time_t)cvtInt64FromZigZag(cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos));
// Checksum page
result.checksumPage = flag & (1 << manifestFilePackFlagChecksumPage) ? true : false;
// SHA1 checksum
memcpy(result.checksumSha1, (const uint8_t *)filePack + bufferPos, HASH_TYPE_SHA1_SIZE_HEX + 1);
bufferPos += HASH_TYPE_SHA1_SIZE_HEX + 1;
// Reference
if (flag & (1 << manifestFilePackFlagReference))
result.reference = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// Mode
result.mode = (mode_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// User/group
result.user = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
result.group = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// Repo size
result.sizeRepo = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// Checksum page error
result.checksumPageError = flag & (1 << manifestFilePackFlagChecksumPageError) ? true : false;
if (flag & (1 << manifestFilePackFlagChecksumPageErrorList))
result.checksumPageErrorList = (const String *)((const uint8_t *)filePack + bufferPos);
FUNCTION_TEST_RETURN(result);
}
void void
manifestFileAdd(Manifest *this, const ManifestFile *file) manifestFileAdd(Manifest *this, ManifestFile file)
{ {
FUNCTION_TEST_BEGIN(); FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST, this); FUNCTION_TEST_PARAM(MANIFEST, this);
FUNCTION_TEST_PARAM(VOID, file);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(file.name != NULL);
file.user = manifestOwnerCache(this, file.user);
file.group = manifestOwnerCache(this, file.group);
if (file.reference != NULL)
file.reference = strLstAddIfMissing(this->referenceList, file.reference);
MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
{
const ManifestFilePack *const filePack = manifestFilePack(&file);
lstAdd(this->pub.fileList, &filePack);
}
MEM_CONTEXT_END();
FUNCTION_TEST_RETURN_VOID();
}
// Update file pack by creating a new one and then freeing the old one
static void
manifestFilePackUpdate(Manifest *const this, ManifestFilePack **const filePack, const ManifestFile *const file)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST, this);
FUNCTION_TEST_PARAM_P(VOID, filePack);
FUNCTION_TEST_PARAM(MANIFEST_FILE, file); FUNCTION_TEST_PARAM(MANIFEST_FILE, file);
FUNCTION_TEST_END(); FUNCTION_TEST_END();
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(filePack != NULL);
ASSERT(file != NULL); ASSERT(file != NULL);
ASSERT(file->name != NULL);
MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList)) MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
{ {
ManifestFile fileAdd = ManifestFilePack *const filePackOld = *filePack;
{ *filePack = manifestFilePack(file);
.checksumPage = file->checksumPage, memFree(filePackOld);
.checksumPageError = file->checksumPageError,
.checksumPageErrorList = varLstDup(file->checksumPageErrorList),
.group = manifestOwnerCache(this, file->group),
.mode = file->mode,
.name = strDup(file->name),
.size = file->size,
.sizeRepo = file->sizeRepo,
.timestamp = file->timestamp,
.user = manifestOwnerCache(this, file->user),
};
memcpy(fileAdd.checksumSha1, file->checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
if (file->reference != NULL)
{
// Search for the reference in the list
for (unsigned int referenceIdx = 0; referenceIdx < strLstSize(this->referenceList); referenceIdx++)
{
const String *found = strLstGet(this->referenceList, referenceIdx);
if (strEq(file->reference, found))
{
fileAdd.reference = found;
break;
}
}
// If not found then add it
if (fileAdd.reference == NULL)
{
strLstAdd(this->referenceList, file->reference);
fileAdd.reference = strLstGet(this->referenceList, strLstSize(this->referenceList) - 1);
}
}
lstAdd(this->pub.fileList, &fileAdd);
} }
MEM_CONTEXT_END(); MEM_CONTEXT_END();
@ -364,7 +513,7 @@ manifestNewInternal(void)
{ {
.memContext = memContextCurrent(), .memContext = memContextCurrent(),
.dbList = lstNewP(sizeof(ManifestDb), .comparator = lstComparatorStr), .dbList = lstNewP(sizeof(ManifestDb), .comparator = lstComparatorStr),
.fileList = lstNewP(sizeof(ManifestFile), .comparator = lstComparatorStr), .fileList = lstNewP(sizeof(ManifestFilePack *), .comparator = lstComparatorStr),
.linkList = lstNewP(sizeof(ManifestLink), .comparator = lstComparatorStr), .linkList = lstNewP(sizeof(ManifestLink), .comparator = lstComparatorStr),
.pathList = lstNewP(sizeof(ManifestPath), .comparator = lstComparatorStr), .pathList = lstNewP(sizeof(ManifestPath), .comparator = lstComparatorStr),
.targetList = lstNewP(sizeof(ManifestTarget), .comparator = lstComparatorStr), .targetList = lstNewP(sizeof(ManifestTarget), .comparator = lstComparatorStr),
@ -843,7 +992,7 @@ manifestBuildCallback(void *data, const StorageInfo *info)
!strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL); !strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL);
} }
manifestFileAdd(buildData.manifest, &file); manifestFileAdd(buildData.manifest, file);
break; break;
} }
@ -1163,12 +1312,12 @@ manifestNewBuild(
while (fileIdx < manifestFileTotal(this)) while (fileIdx < manifestFileTotal(this))
{ {
// If this file looks like a relation. Note that this never matches on _init forks. // If this file looks like a relation. Note that this never matches on _init forks.
const ManifestFile *file = manifestFile(this, fileIdx); const String *const filePathName = manifestFileNameGet(this, fileIdx);
if (regExpMatch(relationExp, file->name)) if (regExpMatch(relationExp, filePathName))
{ {
// Get the filename (without path) // Get the filename (without path)
const char *fileName = strBaseZ(file->name); const char *fileName = strBaseZ(filePathName);
size_t fileNameSize = strlen(fileName); size_t fileNameSize = strlen(fileName);
// Strip off the numeric part of the relation // Strip off the numeric part of the relation
@ -1193,8 +1342,8 @@ manifestNewBuild(
{ {
// Determine if the relation is unlogged // Determine if the relation is unlogged
String *relationInit = strNewFmt( String *relationInit = strNewFmt(
"%.*s%s_init", (int)(strSize(file->name) - fileNameSize), strZ(file->name), relationFileId); "%.*s%s_init", (int)(strSize(filePathName) - fileNameSize), strZ(filePathName), relationFileId);
lastRelationFileIdUnlogged = manifestFileFindDefault(this, relationInit, NULL) != NULL; lastRelationFileIdUnlogged = manifestFileExists(this, relationInit);
strFree(relationInit); strFree(relationInit);
// Save the file id so we don't need to do the lookup next time if it doesn't change // Save the file id so we don't need to do the lookup next time if it doesn't change
@ -1204,7 +1353,7 @@ manifestNewBuild(
// If relation is unlogged then remove it // If relation is unlogged then remove it
if (lastRelationFileIdUnlogged) if (lastRelationFileIdUnlogged)
{ {
manifestFileRemove(this, file->name); manifestFileRemove(this, filePathName);
continue; continue;
} }
} }
@ -1262,13 +1411,13 @@ manifestBuildValidate(Manifest *this, bool delta, time_t copyStart, CompressType
{ {
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
{ {
const ManifestFile *file = manifestFile(this, fileIdx); const ManifestFile file = manifestFile(this, fileIdx);
// Check for timestamp in the future // Check for timestamp in the future
if (file->timestamp > copyStart) if (file.timestamp > copyStart)
{ {
LOG_WARN_FMT( LOG_WARN_FMT(
"file '%s' has timestamp in the future, enabling delta checksum", strZ(manifestPathPg(file->name))); "file '%s' has timestamp in the future, enabling delta checksum", strZ(manifestPathPg(file.name)));
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR; this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
break; break;
@ -1337,29 +1486,30 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
{ {
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
{ {
const ManifestFile *file = manifestFile(this, fileIdx); const ManifestFile file = manifestFile(this, fileIdx);
const ManifestFile *filePrior = manifestFileFindDefault(manifestPrior, file->name, NULL);
// If file was found in prior manifest then perform checks // If file was found in prior manifest then perform checks
if (filePrior != NULL) if (manifestFileExists(manifestPrior, file.name))
{ {
const ManifestFile filePrior = manifestFileFind(manifestPrior, file.name);
// Check for timestamp earlier than the prior backup // Check for timestamp earlier than the prior backup
if (file->timestamp < filePrior->timestamp) if (file.timestamp < filePrior.timestamp)
{ {
LOG_WARN_FMT( LOG_WARN_FMT(
"file '%s' has timestamp earlier than prior backup, enabling delta checksum", "file '%s' has timestamp earlier than prior backup, enabling delta checksum",
strZ(manifestPathPg(file->name))); strZ(manifestPathPg(file.name)));
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR; this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
break; break;
} }
// Check for size change with no timestamp change // Check for size change with no timestamp change
if (file->size != filePrior->size && file->timestamp == filePrior->timestamp) if (file.size != filePrior.size && file.timestamp == filePrior.timestamp)
{ {
LOG_WARN_FMT( LOG_WARN_FMT(
"file '%s' has same timestamp as prior but different size, enabling delta checksum", "file '%s' has same timestamp as prior but different size, enabling delta checksum",
strZ(manifestPathPg(file->name))); strZ(manifestPathPg(file.name)));
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR; this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
break; break;
@ -1375,17 +1525,20 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
for (unsigned int fileIdx = 0; fileIdx < lstSize(this->pub.fileList); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < lstSize(this->pub.fileList); fileIdx++)
{ {
const ManifestFile *file = manifestFile(this, fileIdx); const ManifestFile file = manifestFile(this, fileIdx);
const ManifestFile *filePrior = manifestFileFindDefault(manifestPrior, file->name, NULL);
// Check if prior file can be used // Check if prior file can be used
if (filePrior != NULL && file->size == filePrior->size && if (manifestFileExists(manifestPrior, file.name))
(delta || file->size == 0 || file->timestamp == filePrior->timestamp)) {
const ManifestFile filePrior = manifestFileFind(manifestPrior, file.name);
if (file.size == filePrior.size && (delta || file.size == 0 || file.timestamp == filePrior.timestamp))
{ {
manifestFileUpdate( manifestFileUpdate(
this, file->name, file->size, filePrior->sizeRepo, filePrior->checksumSha1, this, file.name, file.size, filePrior.sizeRepo, filePrior.checksumSha1,
VARSTR(filePrior->reference != NULL ? filePrior->reference : manifestPrior->pub.data.backupLabel), VARSTR(filePrior.reference != NULL ? filePrior.reference : manifestPrior->pub.data.backupLabel),
filePrior->checksumPage, filePrior->checksumPageError, filePrior->checksumPageErrorList); filePrior.checksumPage, filePrior.checksumPageError, filePrior.checksumPageErrorList);
}
} }
} }
} }
@ -1618,7 +1771,7 @@ manifestLoadCallback(void *callbackData, const String *section, const String *ke
const Variant *checksumPageErrorList = kvGetDefault(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, NULL); const Variant *checksumPageErrorList = kvGetDefault(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, NULL);
if (checksumPageErrorList != NULL) if (checksumPageErrorList != NULL)
file.checksumPageErrorList = varVarLst(checksumPageErrorList); file.checksumPageErrorList = jsonFromVar(checksumPageErrorList);
} }
if (kvKeyExists(fileKv, MANIFEST_KEY_GROUP_VAR)) if (kvKeyExists(fileKv, MANIFEST_KEY_GROUP_VAR))
@ -1640,7 +1793,7 @@ manifestLoadCallback(void *callbackData, const String *section, const String *ke
} }
lstAdd(loadData->fileFoundList, &valueFound); lstAdd(loadData->fileFoundList, &valueFound);
manifestFileAdd(manifest, &file); manifestFileAdd(manifest, file);
} }
MEM_CONTEXT_END(); MEM_CONTEXT_END();
} }
@ -1926,17 +2079,20 @@ manifestNewLoad(IoRead *read)
// Process file defaults // Process file defaults
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
{ {
ManifestFile *file = lstGet(this->pub.fileList, fileIdx); ManifestFilePack **const filePack = lstGet(this->pub.fileList, fileIdx);
ManifestFile file = manifestFileUnpack(*filePack);
ManifestLoadFound *found = lstGet(loadData.fileFoundList, fileIdx); ManifestLoadFound *found = lstGet(loadData.fileFoundList, fileIdx);
if (!found->group) if (!found->group)
file->group = manifestOwnerCache(this, manifestOwnerGet(loadData.fileGroupDefault)); file.group = manifestOwnerCache(this, manifestOwnerGet(loadData.fileGroupDefault));
if (!found->mode) if (!found->mode)
file->mode = loadData.fileModeDefault; file.mode = loadData.fileModeDefault;
if (!found->user) if (!found->user)
file->user = manifestOwnerCache(this, manifestOwnerGet(loadData.fileUserDefault)); file.user = manifestOwnerCache(this, manifestOwnerGet(loadData.fileUserDefault));
manifestFilePackUpdate(this, filePack, &file);
} }
// Process link defaults // Process link defaults
@ -2243,41 +2399,41 @@ manifestSaveCallback(void *callbackData, const String *sectionNext, InfoSave *in
{ {
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{ {
const ManifestFile *file = manifestFile(manifest, fileIdx); const ManifestFile file = manifestFile(manifest, fileIdx);
KeyValue *fileKv = kvNew(); KeyValue *fileKv = kvNew();
// Save if the file size is not zero and the checksum exists. The checksum might not exist if this is a partial // Save if the file size is not zero and the checksum exists. The checksum might not exist if this is a partial
// save performed during a backup. // save performed during a backup.
if (file->size != 0 && file->checksumSha1[0] != 0) if (file.size != 0 && file.checksumSha1[0] != 0)
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_VAR, VARSTRZ(file->checksumSha1)); kvPut(fileKv, MANIFEST_KEY_CHECKSUM_VAR, VARSTRZ(file.checksumSha1));
if (file->checksumPage) if (file.checksumPage)
{ {
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, VARBOOL(!file->checksumPageError)); kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, VARBOOL(!file.checksumPageError));
if (file->checksumPageErrorList != NULL) if (file.checksumPageErrorList != NULL)
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, varNewVarLst(file->checksumPageErrorList)); kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, jsonToVar(file.checksumPageErrorList));
} }
if (!varEq(manifestOwnerVar(file->group), saveData->groupDefault)) if (!varEq(manifestOwnerVar(file.group), saveData->groupDefault))
kvPut(fileKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(file->group)); kvPut(fileKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(file.group));
if (file->mode != saveData->fileModeDefault) if (file.mode != saveData->fileModeDefault)
kvPut(fileKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", file->mode))); kvPut(fileKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", file.mode)));
if (file->reference != NULL) if (file.reference != NULL)
kvPut(fileKv, MANIFEST_KEY_REFERENCE_VAR, VARSTR(file->reference)); kvPut(fileKv, MANIFEST_KEY_REFERENCE_VAR, VARSTR(file.reference));
if (file->sizeRepo != file->size) if (file.sizeRepo != file.size)
kvPut(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, VARUINT64(file->sizeRepo)); kvPut(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, VARUINT64(file.sizeRepo));
kvPut(fileKv, MANIFEST_KEY_SIZE_VAR, VARUINT64(file->size)); kvPut(fileKv, MANIFEST_KEY_SIZE_VAR, VARUINT64(file.size));
kvPut(fileKv, MANIFEST_KEY_TIMESTAMP_VAR, VARUINT64((uint64_t)file->timestamp)); kvPut(fileKv, MANIFEST_KEY_TIMESTAMP_VAR, VARUINT64((uint64_t)file.timestamp));
if (!varEq(manifestOwnerVar(file->user), saveData->userDefault)) if (!varEq(manifestOwnerVar(file.user), saveData->userDefault))
kvPut(fileKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(file->user)); kvPut(fileKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(file.user));
infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, file->name, jsonFromKv(fileKv)); infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, file.name, jsonFromKv(fileKv));
MEM_CONTEXT_TEMP_RESET(1000); MEM_CONTEXT_TEMP_RESET(1000);
} }
@ -2437,22 +2593,22 @@ manifestValidate(Manifest *this, bool strict)
// Validate files // Validate files
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
{ {
const ManifestFile *file = manifestFile(this, fileIdx); const ManifestFile file = manifestFile(this, fileIdx);
// All files must have a checksum // All files must have a checksum
if (file->checksumSha1[0] == '\0') if (file.checksumSha1[0] == '\0')
strCatFmt(error, "\nmissing checksum for file '%s'", strZ(file->name)); strCatFmt(error, "\nmissing checksum for file '%s'", strZ(file.name));
// These are strict checks to be performed only after a backup and before the final manifest save // These are strict checks to be performed only after a backup and before the final manifest save
if (strict) if (strict)
{ {
// Zero-length files must have a specific checksum // Zero-length files must have a specific checksum
if (file->size == 0 && !strEqZ(HASH_TYPE_SHA1_ZERO_STR, file->checksumSha1)) if (file.size == 0 && !strEqZ(HASH_TYPE_SHA1_ZERO_STR, file.checksumSha1))
strCatFmt(error, "\ninvalid checksum '%s' for zero size file '%s'", file->checksumSha1, strZ(file->name)); strCatFmt(error, "\ninvalid checksum '%s' for zero size file '%s'", file.checksumSha1, strZ(file.name));
// Non-zero size files must have non-zero repo size // Non-zero size files must have non-zero repo size
if (file->sizeRepo == 0 && file->size != 0) if (file.sizeRepo == 0 && file.size != 0)
strCatFmt(error, "\nrepo size must be > 0 for file '%s'", strZ(file->name)); strCatFmt(error, "\nrepo size must be > 0 for file '%s'", strZ(file.name));
} }
} }
@ -2490,8 +2646,8 @@ manifestDbFind(const Manifest *this, const String *name)
/*********************************************************************************************************************************** /***********************************************************************************************************************************
File functions and getters/setters File functions and getters/setters
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
const ManifestFile * static ManifestFilePack **
manifestFileFind(const Manifest *this, const String *name) manifestFilePackFindInternal(const Manifest *this, const String *name)
{ {
FUNCTION_TEST_BEGIN(); FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST, this); FUNCTION_TEST_PARAM(MANIFEST, this);
@ -2501,12 +2657,26 @@ manifestFileFind(const Manifest *this, const String *name)
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(name != NULL); ASSERT(name != NULL);
const ManifestFile *result = lstFind(this->pub.fileList, &name); ManifestFilePack **const filePack = lstFind(this->pub.fileList, &name);
if (result == NULL) if (filePack == NULL)
THROW_FMT(AssertError, "unable to find '%s' in manifest file list", strZ(name)); THROW_FMT(AssertError, "unable to find '%s' in manifest file list", strZ(name));
FUNCTION_TEST_RETURN(result); FUNCTION_TEST_RETURN(filePack);
}
const ManifestFilePack *
manifestFilePackFind(const Manifest *this, const String *name)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST, this);
FUNCTION_TEST_PARAM(STRING, name);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(name != NULL);
FUNCTION_TEST_RETURN(*manifestFilePackFindInternal(this, name));
} }
void void
@ -2528,8 +2698,9 @@ manifestFileRemove(const Manifest *this, const String *name)
void void
manifestFileUpdate( manifestFileUpdate(
Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference, Manifest *const this, const String *const name, const uint64_t size, const uint64_t sizeRepo, const char *const checksumSha1,
bool checksumPage, bool checksumPageError, const VariantList *checksumPageErrorList) const Variant *const reference, const bool checksumPage, const bool checksumPageError,
const String *const checksumPageErrorList)
{ {
FUNCTION_TEST_BEGIN(); FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST, this); FUNCTION_TEST_PARAM(MANIFEST, this);
@ -2540,7 +2711,7 @@ manifestFileUpdate(
FUNCTION_TEST_PARAM(VARIANT, reference); FUNCTION_TEST_PARAM(VARIANT, reference);
FUNCTION_TEST_PARAM(BOOL, checksumPage); FUNCTION_TEST_PARAM(BOOL, checksumPage);
FUNCTION_TEST_PARAM(BOOL, checksumPageError); FUNCTION_TEST_PARAM(BOOL, checksumPageError);
FUNCTION_TEST_PARAM(VARIANT_LIST, checksumPageErrorList); FUNCTION_TEST_PARAM(STRING, checksumPageErrorList);
FUNCTION_TEST_END(); FUNCTION_TEST_END();
ASSERT(this != NULL); ASSERT(this != NULL);
@ -2549,33 +2720,32 @@ manifestFileUpdate(
(!checksumPage && !checksumPageError && checksumPageErrorList == NULL) || (!checksumPage && !checksumPageError && checksumPageErrorList == NULL) ||
(checksumPage && !checksumPageError && checksumPageErrorList == NULL) || (checksumPage && checksumPageError)); (checksumPage && !checksumPageError && checksumPageErrorList == NULL) || (checksumPage && checksumPageError));
ManifestFile *file = (ManifestFile *)manifestFileFind(this, name); ManifestFilePack **const filePack = manifestFilePackFindInternal(this, name);
ManifestFile file = manifestFileUnpack(*filePack);
MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
{
// Update reference if set // Update reference if set
if (reference != NULL) if (reference != NULL)
{ {
if (varStr(reference) == NULL) if (varStr(reference) == NULL)
file->reference = NULL; file.reference = NULL;
else else
file->reference = strLstAddIfMissing(this->referenceList, varStr(reference)); file.reference = strLstAddIfMissing(this->referenceList, varStr(reference));
} }
// Update checksum if set // Update checksum if set
if (checksumSha1 != NULL) if (checksumSha1 != NULL)
memcpy(file->checksumSha1, checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1); memcpy(file.checksumSha1, checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
// Update repo size // Update repo size
file->size = size; file.size = size;
file->sizeRepo = sizeRepo; file.sizeRepo = sizeRepo;
// Update checksum page info // Update checksum page info
file->checksumPage = checksumPage; file.checksumPage = checksumPage;
file->checksumPageError = checksumPageError; file.checksumPageError = checksumPageError;
file->checksumPageErrorList = varLstDup(checksumPageErrorList); file.checksumPageErrorList = checksumPageErrorList;
}
MEM_CONTEXT_END(); manifestFilePackUpdate(this, filePack, &file);
FUNCTION_TEST_RETURN_VOID(); FUNCTION_TEST_RETURN_VOID();
} }

53
src/info/manifest.h
View File

@ -96,7 +96,7 @@ typedef struct ManifestFile
bool checksumPageError:1; // Is there an error in the page checksum? bool checksumPageError:1; // Is there an error in the page checksum?
mode_t mode; // File mode mode_t mode; // File mode
char checksumSha1[HASH_TYPE_SHA1_SIZE_HEX + 1]; // SHA1 checksum char checksumSha1[HASH_TYPE_SHA1_SIZE_HEX + 1]; // SHA1 checksum
const VariantList *checksumPageErrorList; // List of page checksum errors if there are any const String *checksumPageErrorList; // List of page checksum errors if there are any
const String *user; // User name const String *user; // User name
const String *group; // Group name const String *group; // Group name
const String *reference; // Reference to a prior backup const String *reference; // Reference to a prior backup
@ -258,21 +258,52 @@ manifestDbTotal(const Manifest *const this)
/*********************************************************************************************************************************** /***********************************************************************************************************************************
File functions and getters/setters File functions and getters/setters
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
__attribute__((always_inline)) static inline const ManifestFile * typedef struct ManifestFilePack ManifestFilePack;
manifestFile(const Manifest *const this, const unsigned int fileIdx)
// Unpack file pack returned by manifestFilePackGet()
ManifestFile manifestFileUnpack(const ManifestFilePack *filePack);
// Get file in pack format by index
__attribute__((always_inline)) static inline const ManifestFilePack *
manifestFilePackGet(const Manifest *const this, const unsigned int fileIdx)
{ {
return lstGet(THIS_PUB(Manifest)->fileList, fileIdx); return *(ManifestFilePack **)lstGet(THIS_PUB(Manifest)->fileList, fileIdx);
} }
void manifestFileAdd(Manifest *this, const ManifestFile *file); // Get file name
const ManifestFile *manifestFileFind(const Manifest *this, const String *name); __attribute__((always_inline)) static inline const String *
manifestFileNameGet(const Manifest *const this, const unsigned int fileIdx)
{
return (const String *)manifestFilePackGet(this, fileIdx);
}
// If the file requested is not found in the list, return the default passed rather than throw an error // Get file by index
__attribute__((always_inline)) static inline const ManifestFile * __attribute__((always_inline)) static inline ManifestFile
manifestFileFindDefault(const Manifest *const this, const String *const name, const ManifestFile *const fileDefault) manifestFile(const Manifest *const this, const unsigned int fileIdx)
{
return manifestFileUnpack(manifestFilePackGet(this, fileIdx));
}
// Add a file
void manifestFileAdd(Manifest *this, ManifestFile file);
// Find file in pack format by name
const ManifestFilePack *manifestFilePackFind(const Manifest *this, const String *name);
// Find file by name
__attribute__((always_inline)) static inline ManifestFile
manifestFileFind(const Manifest *const this, const String *const name)
{ {
ASSERT_INLINE(name != NULL); ASSERT_INLINE(name != NULL);
return lstFindDefault(THIS_PUB(Manifest)->fileList, &name, (void *)fileDefault); return manifestFileUnpack(manifestFilePackFind(this, name));
}
// Does the file exist?
__attribute__((always_inline)) static inline bool
manifestFileExists(const Manifest *const this, const String *const name)
{
ASSERT_INLINE(name != NULL);
return lstFindDefault(THIS_PUB(Manifest)->fileList, &name, NULL) != NULL;
} }
void manifestFileRemove(const Manifest *this, const String *name); void manifestFileRemove(const Manifest *this, const String *name);
@ -286,7 +317,7 @@ manifestFileTotal(const Manifest *const this)
// Update a file with new data // Update a file with new data
void manifestFileUpdate( void manifestFileUpdate(
Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference, Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference,
bool checksumPage, bool checksumPageError, const VariantList *checksumPageErrorList); bool checksumPage, bool checksumPageError, const String *checksumPageErrorList);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Link functions and getters/setters Link functions and getters/setters

2
test/define.yaml
View File

@ -73,7 +73,7 @@ unit:
# ---------------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------------
- name: type-convert - name: type-convert
total: 11 total: 12
coverage: coverage:
- common/type/convert - common/type/convert

48
test/src/module/command/backupTest.c
View File

@ -23,7 +23,7 @@ typedef struct TestBackupValidateCallbackData
{ {
const Storage *storage; // Storage object when needed (e.g. fileCompressed = true) const Storage *storage; // Storage object when needed (e.g. fileCompressed = true)
const String *path; // Subpath when storage is specified const String *path; // Subpath when storage is specified
const Manifest *manifest; // Manifest to check for files/links/paths Manifest *manifest; // Manifest to check for files/links/paths
const ManifestData *manifestData; // Manifest data const ManifestData *manifestData; // Manifest data
String *content; // String where content should be added String *content; // String where content should be added
} TestBackupValidateCallbackData; } TestBackupValidateCallbackData;
@ -76,30 +76,31 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
// Check against the manifest // Check against the manifest
// --------------------------------------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------------------------------------
const ManifestFile *file = manifestFileFind(data->manifest, manifestName); ManifestFilePack **const filePack = manifestFilePackFindInternal(data->manifest, manifestName);
ManifestFile file = manifestFileUnpack(*filePack);
// Test size and repo-size. If compressed then set the repo-size to size so it will not be in test output. Even the same // Test size and repo-size. If compressed then set the repo-size to size so it will not be in test output. Even the same
// compression algorithm can give slightly different results based on the version so repo-size is not deterministic for // compression algorithm can give slightly different results based on the version so repo-size is not deterministic for
// compression. // compression.
if (size != file->size) if (size != file.size)
THROW_FMT(AssertError, "'%s' size does match manifest", strZ(manifestName)); THROW_FMT(AssertError, "'%s' size does match manifest", strZ(manifestName));
if (info->size != file->sizeRepo) if (info->size != file.sizeRepo)
THROW_FMT(AssertError, "'%s' repo size does match manifest", strZ(manifestName)); THROW_FMT(AssertError, "'%s' repo size does match manifest", strZ(manifestName));
if (data->manifestData->backupOptionCompressType != compressTypeNone) if (data->manifestData->backupOptionCompressType != compressTypeNone)
((ManifestFile *)file)->sizeRepo = file->size; file.sizeRepo = file.size;
// Test the checksum. pg_control and WAL headers have different checksums depending on cpu architecture so remove // Test the checksum. pg_control and WAL headers have different checksums depending on cpu architecture so remove
// the checksum from the test output. // the checksum from the test output.
if (!strEqZ(checksum, file->checksumSha1)) if (!strEqZ(checksum, file.checksumSha1))
THROW_FMT(AssertError, "'%s' checksum does match manifest", strZ(manifestName)); THROW_FMT(AssertError, "'%s' checksum does match manifest", strZ(manifestName));
if (strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL) || if (strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL) ||
strBeginsWith( strBeginsWith(
manifestName, strNewFmt(MANIFEST_TARGET_PGDATA "/%s/", strZ(pgWalPath(data->manifestData->pgVersion))))) manifestName, strNewFmt(MANIFEST_TARGET_PGDATA "/%s/", strZ(pgWalPath(data->manifestData->pgVersion)))))
{ {
((ManifestFile *)file)->checksumSha1[0] = '\0'; file.checksumSha1[0] = '\0';
} }
// Test mode, user, group. These values are not in the manifest but we know what they should be based on the default // Test mode, user, group. These values are not in the manifest but we know what they should be based on the default
@ -113,6 +114,9 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
if (!strEq(info->group, TEST_GROUP_STR)) if (!strEq(info->group, TEST_GROUP_STR))
THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(manifestName)); THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(manifestName));
// Update changes to manifest file
manifestFilePackUpdate(data->manifest, filePack, &file);
break; break;
} }
@ -1310,7 +1314,7 @@ testRun(void)
manifestTargetAdd(manifestResume, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")}); manifestTargetAdd(manifestResume, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")});
manifestPathAdd(manifestResume, &(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR}); manifestPathAdd(manifestResume, &(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR});
manifestFileAdd(manifestResume, &(ManifestFile){.name = STRDEF("pg_data/" PG_FILE_PGVERSION)}); manifestFileAdd(manifestResume, (ManifestFile){.name = STRDEF("pg_data/" PG_FILE_PGVERSION)});
manifestSave( manifestSave(
manifestResume, manifestResume,
@ -1430,7 +1434,7 @@ testRun(void)
OBJ_NEW_BEGIN(Manifest) OBJ_NEW_BEGIN(Manifest)
{ {
manifest = manifestNewInternal(); manifest = manifestNewInternal();
manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF("pg_data/test")}); manifestFileAdd(manifest, (ManifestFile){.name = STRDEF("pg_data/test")});
} }
OBJ_NEW_END(); OBJ_NEW_END();
@ -1748,9 +1752,12 @@ testRun(void)
storagePg(), PG_FILE_PGVERSION, storageRepoWrite(), storagePg(), PG_FILE_PGVERSION, storageRepoWrite(),
strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION", strZ(resumeLabel)))); strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION", strZ(resumeLabel))));
strcpy( ManifestFilePack **const filePack = manifestFilePackFindInternal(manifestResume, STRDEF("pg_data/PG_VERSION"));
((ManifestFile *)manifestFileFind(manifestResume, STRDEF("pg_data/PG_VERSION")))->checksumSha1, ManifestFile file = manifestFileUnpack(*filePack);
"06d06bb31b570b94d7b4325f511f853dbe771c21");
strcpy(file.checksumSha1, "06d06bb31b570b94d7b4325f511f853dbe771c21");
manifestFilePackUpdate(manifestResume, filePack, &file);
// Save the resume manifest // Save the resume manifest
manifestSave( manifestSave(
@ -1841,14 +1848,19 @@ testRun(void)
HRN_STORAGE_PUT_EMPTY( HRN_STORAGE_PUT_EMPTY(
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/global/pg_control.gz", strZ(resumeLabel)))); storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/global/pg_control.gz", strZ(resumeLabel))));
((ManifestFile *)manifestFileFind(manifestResume, STRDEF("pg_data/global/pg_control")))->checksumSha1[0] = 0; ManifestFilePack **const filePack = manifestFilePackFindInternal(manifestResume, STRDEF("pg_data/global/pg_control"));
ManifestFile file = manifestFileUnpack(*filePack);
file.checksumSha1[0] = 0;
manifestFilePackUpdate(manifestResume, filePack, &file);
// Size does not match between cluster and resume manifest // Size does not match between cluster and resume manifest
HRN_STORAGE_PUT_Z(storagePgWrite(), "size-mismatch", "TEST", .timeModified = backupTimeStart); HRN_STORAGE_PUT_Z(storagePgWrite(), "size-mismatch", "TEST", .timeModified = backupTimeStart);
HRN_STORAGE_PUT_EMPTY( HRN_STORAGE_PUT_EMPTY(
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/size-mismatch.gz", strZ(resumeLabel)))); storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/size-mismatch.gz", strZ(resumeLabel))));
manifestFileAdd( manifestFileAdd(
manifestResume, &(ManifestFile){ manifestResume, (ManifestFile){
.name = STRDEF("pg_data/size-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", .name = STRDEF("pg_data/size-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
.size = 33}); .size = 33});
@ -1857,7 +1869,7 @@ testRun(void)
HRN_STORAGE_PUT_EMPTY( HRN_STORAGE_PUT_EMPTY(
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch.gz", strZ(resumeLabel)))); storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch.gz", strZ(resumeLabel))));
manifestFileAdd( manifestFileAdd(
manifestResume, &(ManifestFile){ manifestResume, (ManifestFile){
.name = STRDEF("pg_data/time-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", .size = 4, .name = STRDEF("pg_data/time-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", .size = 4,
.timestamp = backupTimeStart - 1}); .timestamp = backupTimeStart - 1});
@ -1867,7 +1879,7 @@ testRun(void)
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/zero-size.gz", strZ(resumeLabel))), storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/zero-size.gz", strZ(resumeLabel))),
"ZERO-SIZE"); "ZERO-SIZE");
manifestFileAdd( manifestFileAdd(
manifestResume, &(ManifestFile){.name = STRDEF("pg_data/zero-size"), .size = 0, .timestamp = backupTimeStart}); manifestResume, (ManifestFile){.name = STRDEF("pg_data/zero-size"), .size = 0, .timestamp = backupTimeStart});
// Path is not in manifest // Path is not in manifest
HRN_STORAGE_PATH_CREATE( HRN_STORAGE_PATH_CREATE(
@ -2028,7 +2040,7 @@ testRun(void)
HRN_STORAGE_PUT_EMPTY( HRN_STORAGE_PUT_EMPTY(
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/resume-ref.gz", strZ(resumeLabel)))); storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/resume-ref.gz", strZ(resumeLabel))));
manifestFileAdd( manifestFileAdd(
manifestResume, &(ManifestFile){.name = STRDEF("pg_data/resume-ref"), .size = 0, .reference = STRDEF("BOGUS")}); manifestResume, (ManifestFile){.name = STRDEF("pg_data/resume-ref"), .size = 0, .reference = STRDEF("BOGUS")});
// Time does not match between cluster and resume manifest (but resume because time is in future so delta enabled). Note // Time does not match between cluster and resume manifest (but resume because time is in future so delta enabled). Note
// also that the repo file is intenionally corrupt to generate a warning about corruption in the repository. // also that the repo file is intenionally corrupt to generate a warning about corruption in the repository.
@ -2036,7 +2048,7 @@ testRun(void)
HRN_STORAGE_PUT_EMPTY( HRN_STORAGE_PUT_EMPTY(
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch2.gz", strZ(resumeLabel)))); storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch2.gz", strZ(resumeLabel))));
manifestFileAdd( manifestFileAdd(
manifestResume, &(ManifestFile){ manifestResume, (ManifestFile){
.name = STRDEF("pg_data/time-mismatch2"), .checksumSha1 = "984816fd329622876e14907634264e6f332e9fb3", .size = 4, .name = STRDEF("pg_data/time-mismatch2"), .checksumSha1 = "984816fd329622876e14907634264e6f332e9fb3", .size = 4,
.timestamp = backupTimeStart}); .timestamp = backupTimeStart});

58
test/src/module/command/restoreTest.c
View File

@ -131,7 +131,7 @@ testManifestMinimal(const String *label, unsigned int pgVersion, const String *p
manifestPathAdd(result, &pathBase); manifestPathAdd(result, &pathBase);
ManifestFile fileVersion = { ManifestFile fileVersion = {
.name = STRDEF("pg_data/" PG_FILE_PGVERSION), .mode = 0600, .group = groupName(), .user = userName()}; .name = STRDEF("pg_data/" PG_FILE_PGVERSION), .mode = 0600, .group = groupName(), .user = userName()};
manifestFileAdd(result, &fileVersion); manifestFileAdd(result, fileVersion);
} }
OBJ_NEW_END(); OBJ_NEW_END();
@ -815,7 +815,7 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/pg_hba.conf"), .size = 4, .timestamp = 1482182860}); (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/pg_hba.conf"), .size = 4, .timestamp = 1482182860});
TEST_RESULT_VOID(restoreManifestMap(manifest), "remap links"); TEST_RESULT_VOID(restoreManifestMap(manifest), "remap links");
@ -982,7 +982,7 @@ testRun(void)
ManifestPath path = {.name = STRDEF("pg_data/bogus_path"), .user = STRDEF("path-user-bogus")}; ManifestPath path = {.name = STRDEF("pg_data/bogus_path"), .user = STRDEF("path-user-bogus")};
manifestPathAdd(manifest, &path); manifestPathAdd(manifest, &path);
ManifestFile file = {.name = STRDEF("pg_data/bogus_file"), .mode = 0600, .group = STRDEF("file-group-bogus")}; ManifestFile file = {.name = STRDEF("pg_data/bogus_file"), .mode = 0600, .group = STRDEF("file-group-bogus")};
manifestFileAdd(manifest, &file); manifestFileAdd(manifest, file);
ManifestLink link = {.name = STRDEF("pg_data/bogus_link"), .destination = STRDEF("/"), .group = STRDEF("link-group-bogus")}; ManifestLink link = {.name = STRDEF("pg_data/bogus_link"), .destination = STRDEF("/"), .group = STRDEF("link-group-bogus")};
manifestLinkAdd(manifest, &link); manifestLinkAdd(manifest, &link);
@ -1034,7 +1034,7 @@ testRun(void)
manifest = testManifestMinimal(STRDEF("20161219-212741F_20161219-21275D"), PG_VERSION_96, pgPath); manifest = testManifestMinimal(STRDEF("20161219-212741F_20161219-21275D"), PG_VERSION_96, pgPath);
manifestFileAdd(manifest, &file); manifestFileAdd(manifest, file);
manifestLinkAdd(manifest, &link); manifestLinkAdd(manifest, &link);
TEST_RESULT_VOID(restoreManifestOwner(manifest, &rootReplaceUser, &rootReplaceGroup), "check ownership"); TEST_RESULT_VOID(restoreManifestOwner(manifest, &rootReplaceUser, &rootReplaceGroup), "check ownership");
@ -1051,7 +1051,7 @@ testRun(void)
TEST_TITLE("owner is root and ownership of pg_data is bad"); TEST_TITLE("owner is root and ownership of pg_data is bad");
manifestPathAdd(manifest, &path); manifestPathAdd(manifest, &path);
manifestFileAdd(manifest, &file); manifestFileAdd(manifest, file);
HRN_SYSTEM_FMT("sudo chown 77777:77777 %s", strZ(pgPath)); HRN_SYSTEM_FMT("sudo chown 77777:77777 %s", strZ(pgPath));
@ -1243,7 +1243,7 @@ testRun(void)
HRN_SYSTEM_FMT("rm -rf %s/*", strZ(pgPath)); HRN_SYSTEM_FMT("rm -rf %s/*", strZ(pgPath));
manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)}); manifestFileAdd(manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)});
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_POSTGRESQLAUTOCONF); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_POSTGRESQLAUTOCONF);
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_RECOVERYSIGNAL); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_RECOVERYSIGNAL);
@ -1303,7 +1303,7 @@ testRun(void)
manifest->pub.data.pgCatalogVersion = hrnPgCatalogVersion(PG_VERSION_90); manifest->pub.data.pgCatalogVersion = hrnPgCatalogVersion(PG_VERSION_90);
manifestTargetAdd(manifest, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")}); manifestTargetAdd(manifest, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")});
manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION)}); manifestFileAdd(manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION)});
} }
OBJ_NEW_END(); OBJ_NEW_END();
@ -1324,11 +1324,11 @@ testRun(void)
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("user-made-system-db"), .id = 16380, .lastSystemId = 12168}); manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("user-made-system-db"), .id = 16380, .lastSystemId = 12168});
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF(UTF8_DB_NAME), .id = 16384, .lastSystemId = 12168}); manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF(UTF8_DB_NAME), .id = 16384, .lastSystemId = 12168});
manifestFileAdd( manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/1/" PG_FILE_PGVERSION)}); manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/1/" PG_FILE_PGVERSION)});
manifestFileAdd( manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16381/" PG_FILE_PGVERSION)}); manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16381/" PG_FILE_PGVERSION)});
manifestFileAdd( manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16385/" PG_FILE_PGVERSION)}); manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16385/" PG_FILE_PGVERSION)});
} }
MEM_CONTEXT_END(); MEM_CONTEXT_END();
@ -1353,7 +1353,7 @@ testRun(void)
MEM_CONTEXT_BEGIN(manifest->pub.memContext) MEM_CONTEXT_BEGIN(manifest->pub.memContext)
{ {
manifestFileAdd( manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16384/" PG_FILE_PGVERSION)}); manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16384/" PG_FILE_PGVERSION)});
} }
MEM_CONTEXT_END(); MEM_CONTEXT_END();
@ -1424,7 +1424,7 @@ testRun(void)
{ {
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test2"), .id = 32768, .lastSystemId = 12168}); manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test2"), .id = 32768, .lastSystemId = 12168});
manifestFileAdd( manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)}); manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)});
} }
MEM_CONTEXT_END(); MEM_CONTEXT_END();
@ -1448,7 +1448,7 @@ testRun(void)
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387"), .tablespaceId = 16387, .tablespaceName = STRDEF("ts1"), .name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387"), .tablespaceId = 16387, .tablespaceName = STRDEF("ts1"),
.path = STRDEF("/ts1")}); .path = STRDEF("/ts1")});
manifestFileAdd( manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)}); manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)});
} }
MEM_CONTEXT_END(); MEM_CONTEXT_END();
@ -1470,7 +1470,7 @@ testRun(void)
{ {
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test3"), .id = 65536, .lastSystemId = 12168}); manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test3"), .id = 65536, .lastSystemId = 12168});
manifestFileAdd( manifestFileAdd(
manifest, &(ManifestFile){ manifest, (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387/PG_9.4_201409291/65536/" PG_FILE_PGVERSION)}); .name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387/PG_9.4_201409291/65536/" PG_FILE_PGVERSION)});
} }
MEM_CONTEXT_END(); MEM_CONTEXT_END();
@ -2053,7 +2053,7 @@ testRun(void)
// PG_VERSION // PG_VERSION
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"}); .checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"});
@ -2184,7 +2184,7 @@ testRun(void)
// tablespace_map (will be ignored during restore) // tablespace_map (will be ignored during restore)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO}); .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP); HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP);
@ -2218,7 +2218,7 @@ testRun(void)
// pg_tblspc/1/16384/PG_VERSION // pg_tblspc/1/16384/PG_VERSION
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/1/16384/" PG_FILE_PGVERSION), .size = 4, .name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/1/16384/" PG_FILE_PGVERSION), .size = 4,
.timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"}); .checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"});
@ -2400,7 +2400,7 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL), .size = 8192, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL), .size = 8192, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "5e2b96c19c4f5c63a5afa2de504d29fe64a4c908"}); .checksumSha1 = "5e2b96c19c4f5c63a5afa2de504d29fe64a4c908"});
@ -2409,7 +2409,7 @@ testRun(void)
// global/999 // global/999
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/999"), .size = 0, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/999"), .size = 0, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = HASH_TYPE_SHA1_ZERO, .reference = STRDEF(TEST_LABEL)}); .checksumSha1 = HASH_TYPE_SHA1_ZERO, .reference = STRDEF(TEST_LABEL)});
@ -2418,7 +2418,7 @@ testRun(void)
// PG_VERSION // PG_VERSION
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"}); .checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
@ -2439,7 +2439,7 @@ testRun(void)
// base/1/PG_VERSION // base/1/PG_VERSION
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA "base/1/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"}); .checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
@ -2452,7 +2452,7 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/2"), .size = 8192, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA "base/1/2"), .size = 8192, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "4d7b2a36c5387decf799352a3751883b7ceb96aa"}); .checksumSha1 = "4d7b2a36c5387decf799352a3751883b7ceb96aa"});
@ -2470,7 +2470,7 @@ testRun(void)
// base/16384/PG_VERSION // base/16384/PG_VERSION
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA "base/16384/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA "base/16384/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"}); .checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
@ -2483,7 +2483,7 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA "base/16384/16385"), .size = 16384, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA "base/16384/16385"), .size = 16384, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "d74e5f7ebe52a3ed468ba08c5b6aefaccd1ca88f"}); .checksumSha1 = "d74e5f7ebe52a3ed468ba08c5b6aefaccd1ca88f"});
@ -2499,7 +2499,7 @@ testRun(void)
// base/32768/PG_VERSION // base/32768/PG_VERSION
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA "base/32768/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA "base/32768/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"}); .checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
@ -2512,7 +2512,7 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA "base/32768/32769"), .size = 32768, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA "base/32768/32769"), .size = 32768, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "a40f0986acb1531ce0cc75a23dcf8aa406ae9081"}); .checksumSha1 = "a40f0986acb1531ce0cc75a23dcf8aa406ae9081"});
@ -2529,7 +2529,7 @@ testRun(void)
.name = name, .destination = STRDEF("../config/postgresql.conf"), .group = groupName(), .user = userName()}); .name = name, .destination = STRDEF("../config/postgresql.conf"), .group = groupName(), .user = userName()});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA "postgresql.conf"), .size = 15, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA "postgresql.conf"), .size = 15, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "98b8abb2e681e2a5a7d8ab082c0a79727887558d"}); .checksumSha1 = "98b8abb2e681e2a5a7d8ab082c0a79727887558d"});
@ -2546,7 +2546,7 @@ testRun(void)
.name = name, .destination = STRDEF("../config/pg_hba.conf"), .group = groupName(), .user = userName()}); .name = name, .destination = STRDEF("../config/pg_hba.conf"), .group = groupName(), .user = userName()});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA "pg_hba.conf"), .size = 11, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA "pg_hba.conf"), .size = 11, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "401215e092779574988a854d8c7caed7f91dba4b"}); .checksumSha1 = "401215e092779574988a854d8c7caed7f91dba4b"});
@ -2555,7 +2555,7 @@ testRun(void)
// tablespace_map (will be ignored during restore) // tablespace_map (will be ignored during restore)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO}); .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP); HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP);

31
test/src/module/common/typeConvertTest.c
View File

@ -202,5 +202,36 @@ testRun(void)
TEST_RESULT_UINT(cvtZToUInt64("18446744073709551615"), 18446744073709551615U, "convert string to uint64"); TEST_RESULT_UINT(cvtZToUInt64("18446744073709551615"), 18446744073709551615U, "convert string to uint64");
} }
// *****************************************************************************************************************************
if (testBegin("cvtUInt64ToVarInt128() and cvtUInt64FromVarInt128()"))
{
uint8_t buffer[CVT_VARINT128_BUFFER_SIZE];
size_t bufferPos;
bufferPos = 0;
TEST_ERROR(cvtUInt64ToVarInt128(9999, buffer, &bufferPos, 1), AssertError, "buffer overflow");
bufferPos = 0;
TEST_RESULT_VOID(cvtUInt64ToVarInt128(0xFFFFFFFFFFFFFFFF, buffer, &bufferPos, sizeof(buffer)), "to varint-128");
TEST_RESULT_UINT(bufferPos, 10, "check buffer position");
bufferPos = 0;
TEST_RESULT_UINT(cvtUInt64FromVarInt128(buffer, &bufferPos), 0xFFFFFFFFFFFFFFFF, "to uint64");
TEST_RESULT_UINT(bufferPos, 10, "check buffer position");
bufferPos = 0;
TEST_RESULT_VOID(cvtUInt64ToVarInt128(0, buffer, &bufferPos, sizeof(buffer)), "to varint-128");
TEST_RESULT_UINT(bufferPos, 1, "check buffer position");
bufferPos = 0;
TEST_RESULT_UINT(cvtUInt64FromVarInt128(buffer, &bufferPos), 0, "to uint64");
TEST_RESULT_UINT(bufferPos, 1, "check buffer position");
uint8_t buffer2[CVT_VARINT128_BUFFER_SIZE + 1];
memset(buffer2, 0xFF, sizeof(buffer2));
bufferPos = 0;
TEST_ERROR(cvtUInt64FromVarInt128(buffer2, &bufferPos), FormatError, "unterminated varint-128 integer");
}
FUNCTION_HARNESS_RETURN_VOID(); FUNCTION_HARNESS_RETURN_VOID();
} }

50
test/src/module/info/manifestTest.c
View File

@ -950,7 +950,7 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860}); (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860});
TEST_RESULT_VOID(manifestBuildValidate(manifest, false, 1482182860, false), "validate manifest"); TEST_RESULT_VOID(manifestBuildValidate(manifest, false, 1482182860, false), "validate manifest");
TEST_RESULT_INT(manifest->pub.data.backupTimestampCopyStart, 1482182860, "check copy start"); TEST_RESULT_INT(manifest->pub.data.backupTimestampCopyStart, 1482182860, "check copy start");
@ -1029,22 +1029,22 @@ testRun(void)
&(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR, .mode = 0700, .group = STRDEF("test"), .user = STRDEF("test")}); &(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR, .mode = 0700, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/BOGUS"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/BOGUS"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182861, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
} }
@ -1059,17 +1059,17 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifestPrior, manifestPrior,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.checksumSha1 = "da39a3ee5e6b4b0d3255bfef95601890afd80709"}); .checksumSha1 = "da39a3ee5e6b4b0d3255bfef95601890afd80709"});
manifestFileAdd( manifestFileAdd(
manifestPrior, manifestPrior,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182860,
.checksumSha1 = "ccccccccccaaaaaaaaaabbbbbbbbbbdddddddddd"}); .checksumSha1 = "ccccccccccaaaaaaaaaabbbbbbbbbbdddddddddd"});
manifestFileAdd( manifestFileAdd(
manifestPrior, manifestPrior,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"}); .checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
} }
@ -1109,18 +1109,18 @@ testRun(void)
lstClear(manifest->pub.fileList); lstClear(manifest->pub.fileList);
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifestPrior, manifestPrior,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"), .reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"}); .checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
@ -1160,7 +1160,7 @@ testRun(void)
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182859, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182859,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
@ -1171,11 +1171,11 @@ testRun(void)
varLstAdd(checksumPageErrorList, varNewUInt(77)); varLstAdd(checksumPageErrorList, varNewUInt(77));
manifestFileAdd( manifestFileAdd(
manifestPrior, manifestPrior,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"), .reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", .checksumPage = true, .checksumPageError = true, .checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", .checksumPage = true, .checksumPageError = true,
.checksumPageErrorList = checksumPageErrorList}); .checksumPageErrorList = jsonFromVar(varNewVarLst(checksumPageErrorList))});
TEST_RESULT_VOID(manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, NULL), "incremental manifest"); TEST_RESULT_VOID(manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, NULL), "incremental manifest");
@ -1211,18 +1211,18 @@ testRun(void)
lstClear(manifest->pub.fileList); lstClear(manifest->pub.fileList);
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd( manifestFileAdd(
manifestPrior, manifestPrior,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"), .reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"}); .checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
@ -1279,14 +1279,14 @@ testRun(void)
lstClear(manifest->pub.fileList); lstClear(manifest->pub.fileList);
manifestFileAdd( manifestFileAdd(
manifest, manifest,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifest->pub.data.backupOptionOnline = BOOL_TRUE_VAR; manifest->pub.data.backupOptionOnline = BOOL_TRUE_VAR;
manifestFileAdd( manifestFileAdd(
manifestPrior, manifestPrior,
&(ManifestFile){ (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"}); .checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
@ -1729,18 +1729,14 @@ testRun(void)
TEST_TITLE("manifest getters"); TEST_TITLE("manifest getters");
// ManifestFile getters // ManifestFile getters
const ManifestFile *file = NULL; ManifestFile file = {0};
TEST_ERROR(manifestFileFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest file list"); TEST_ERROR(manifestFileFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest file list");
TEST_ASSIGN(file, manifestFileFind(manifest, STRDEF("pg_data/PG_VERSION")), "manifestFileFind()"); TEST_ASSIGN(file, manifestFileFind(manifest, STRDEF("pg_data/PG_VERSION")), "manifestFileFind()");
TEST_RESULT_STR_Z(file->name, "pg_data/PG_VERSION", "find file"); TEST_RESULT_STR_Z(file.name, "pg_data/PG_VERSION", "find file");
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
manifestFileFindDefault(manifest, STRDEF("bogus"), file)->name, "pg_data/PG_VERSION", manifestFileFind(manifest, STRDEF("pg_data/special-@#!$^&*()_+~`{}[]\\:;")).name,
"manifestFileFindDefault() - return default");
TEST_RESULT_STR_Z(
manifestFileFind(manifest, STRDEF("pg_data/special-@#!$^&*()_+~`{}[]\\:;"))->name,
"pg_data/special-@#!$^&*()_+~`{}[]\\:;", "find special file"); "pg_data/special-@#!$^&*()_+~`{}[]\\:;", "find special file");
TEST_ASSIGN(file, manifestFileFindDefault(manifest, STRDEF("bogus"), NULL), "manifestFileFindDefault()"); TEST_RESULT_BOOL(manifestFileExists(manifest, STRDEF("bogus")), false, "manifest file does not exist");
TEST_RESULT_PTR(file, NULL, "return default NULL");
TEST_RESULT_VOID( TEST_RESULT_VOID(
manifestFileUpdate(manifest, STRDEF("pg_data/postgresql.conf"), 4457, 4457, "", NULL, false, false, NULL), manifestFileUpdate(manifest, STRDEF("pg_data/postgresql.conf"), 4457, 4457, "", NULL, false, false, NULL),

4
test/src/module/performance/typeTest.c
View File

@ -317,8 +317,8 @@ testRun(void)
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{ {
const ManifestFile *file = manifestFile(manifest, fileIdx); const ManifestFile file = manifestFile(manifest, fileIdx);
ASSERT(file == manifestFileFind(manifest, file->name)); ASSERT(strEq(file.name, manifestFileFind(manifest, file.name).name));
} }
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin)); TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));