db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-03 14:52:21 +02:00
Code Issues Releases Activity

Pack manifest file structs to save memory.

Browse Source 
Manifests with a very large number of files can use a considerable amount of memory. There are a lot of zeroes in the data so it can be stored more efficiently by using base-128 varint encoding for the integers and storing the strings in the same allocation.

The downside is that the data needs to be unpacked in order to be used, but in most cases this seems fast enough (about 10% slower than before) except for saving the manifest, which is 10% slower up to 10 million files and then gets about 5x slower by 100 million (two minutes on my M1 Mac). Profiling does not show this slowdown so I wonder if this is related to the change in memory layout. Curiously, the function that increased most was jsonFromStrInternal(), which was not modified. That gives more weight to the idea that there is some kind of memory issue going on here and one hopes that servers would be less affected. Either way, they largest use cases we have seen are for about 6 million files so if we can improve that case I believe we will be better off.

Further analysis showed that most of the time was taken up writing the size and timestamp fields, which makes almost no sense. The same amount of time was used if they were hard-coded to 0, which points to some odd memory issue on the M1 architecture.

This change has been planned for a while, but the particular impetus at this time is that small file support requires additional fields that would increase manifest memory usage by about 20%, even if the feature is not used.

Note that the Pack code has been updated to use the new varint encoder, but the decoder remains separate because it needs to fetch one byte at a time.
This commit is contained in:
David Steele 2022-01-21 17:05:07 -05:00 committed by GitHub
parent 575ae77c0d
commit 61ce58692f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 698 additions and 377 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/xml/release.xml
View File

@ -68,6 +68,7 @@
<commit subject="Simplify base path mode in mock/all integration tests."/>
<commit subject="Simplify manifest defaults."/>
<commit subject="Convert varNewUInt64() to VARUINT64() where possible in manifest."/>
<commit subject="Pack manifest file structs to save memory."/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>

131
src/command/backup/backup.c
View File

@ -576,37 +576,43 @@ void backupResumeCallback(void *data, const StorageInfo *info)
if (fileCompressType != compressTypeNone)
manifestName = compressExtStrip(manifestName, fileCompressType);
// Find the file in both manifests
const ManifestFile *file = manifestFileFindDefault(resumeData->manifest, manifestName, NULL);
const ManifestFile *fileResume = manifestFileFindDefault(resumeData->manifestResume, manifestName, NULL);
// Check if the file can be resumed or must be removed
const char *removeReason = NULL;
if (fileCompressType != resumeData->compressType)
removeReason = "mismatched compression type";
else if (file == NULL)
else if (!manifestFileExists(resumeData->manifest, manifestName))
removeReason = "missing in manifest";
else if (file->reference != NULL)
removeReason = "reference in manifest";
else if (fileResume == NULL)
removeReason = "missing in resumed manifest";
else if (fileResume->reference != NULL)
removeReason = "reference in resumed manifest";
else if (fileResume->checksumSha1[0] == '\0')
removeReason = "no checksum in resumed manifest";
else if (file->size != fileResume->size)
removeReason = "mismatched size";
else if (!resumeData->delta && file->timestamp != fileResume->timestamp)
removeReason = "mismatched timestamp";
else if (file->size == 0)
// ??? don't resume zero size files because Perl wouldn't -- this can be removed after the migration)
removeReason = "zero size";
else
{
manifestFileUpdate(
resumeData->manifest, manifestName, file->size, fileResume->sizeRepo, fileResume->checksumSha1, NULL,
fileResume->checksumPage, fileResume->checksumPageError, fileResume->checksumPageErrorList);
const ManifestFile file = manifestFileFind(resumeData->manifest, manifestName);
if (file.reference != NULL)
removeReason = "reference in manifest";
else if (!manifestFileExists(resumeData->manifestResume, manifestName))
removeReason = "missing in resumed manifest";
else
{
const ManifestFile fileResume = manifestFileFind(resumeData->manifestResume, manifestName);
if (fileResume.reference != NULL)
removeReason = "reference in resumed manifest";
else if (fileResume.checksumSha1[0] == '\0')
removeReason = "no checksum in resumed manifest";
else if (file.size != fileResume.size)
removeReason = "mismatched size";
else if (!resumeData->delta && file.timestamp != fileResume.timestamp)
removeReason = "mismatched timestamp";
else if (file.size == 0)
// ??? don't resume zero size files because Perl wouldn't -- this can be removed after the migration)
removeReason = "zero size";
else
{
manifestFileUpdate(
resumeData->manifest, manifestName, file.size, fileResume.sizeRepo, fileResume.checksumSha1, NULL,
fileResume.checksumPage, fileResume.checksumPageError, fileResume.checksumPageErrorList);
}
}
}
// Remove the file if it could not be resumed
@ -981,7 +987,7 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti
file.checksumSha1, strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))),
HASH_TYPE_SHA1_SIZE_HEX + 1);
manifestFileAdd(manifest, &file);
manifestFileAdd(manifest, file);
LOG_DETAIL_FMT("wrote '%s' file returned from pg_stop_backup()", strZ(name));
}
@ -1165,7 +1171,7 @@ backupJobResult(
{
MEM_CONTEXT_TEMP_BEGIN()
{
const ManifestFile *const file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
const ManifestFile file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
const unsigned int processId = protocolParallelJobProcessId(job);
PackRead *const jobResult = protocolParallelJobResult(job);
@ -1205,7 +1211,7 @@ backupJobResult(
else if (copyResult == backupCopyResultSkip)
{
LOG_DETAIL_PID_FMT(processId, "skip file removed by database %s", strZ(fileLog));
strLstAdd(fileRemove, file->name);
strLstAdd(fileRemove, file.name);
}
// Else file was copied so update manifest
else
@ -1220,13 +1226,13 @@ backupJobResult(
" continue but this may be an issue unless the resumed backup path in the repository is known to be"
" corrupted.\n"
"NOTE: this does not indicate a problem with the PostgreSQL page checksums.",
strZ(file->name), file->checksumSha1);
strZ(file.name), file.checksumSha1);
}
LOG_DETAIL_PID_FMT(processId, "backup file %s (%s)%s", strZ(fileLog), strZ(logProgress), strZ(logChecksum));
// If the file had page checksums calculated during the copy
ASSERT((!file->checksumPage && checksumPageResult == NULL) || (file->checksumPage && checksumPageResult != NULL));
ASSERT((!file.checksumPage && checksumPageResult == NULL) || (file.checksumPage && checksumPageResult != NULL));
bool checksumPageError = false;
const VariantList *checksumPageErrorList = NULL;
@ -1300,8 +1306,8 @@ backupJobResult(
// Update file info and remove any reference to the file's existence in a prior backup
manifestFileUpdate(
manifest, file->name, copySize, repoSize, strZ(copyChecksum), VARSTR(NULL), file->checksumPage,
checksumPageError, checksumPageErrorList);
manifest, file.name, copySize, repoSize, strZ(copyChecksum), VARSTR(NULL), file.checksumPage,
checksumPageError, checksumPageErrorList != NULL ? jsonFromVar(varNewVarLst(checksumPageErrorList)) : NULL);
}
}
MEM_CONTEXT_TEMP_END();
@ -1420,14 +1426,18 @@ backupProcessQueueComparator(const void *item1, const void *item2)
ASSERT(item1 != NULL);
ASSERT(item2 != NULL);
// Unpack files
ManifestFile file1 = manifestFileUnpack(*(const ManifestFilePack **)item1);
ManifestFile file2 = manifestFileUnpack(*(const ManifestFilePack **)item2);
// If the size differs then that's enough to determine order
if ((*(ManifestFile **)item1)->size < (*(ManifestFile **)item2)->size)
if (file1.size < file2.size)
FUNCTION_TEST_RETURN(-1);
else if ((*(ManifestFile **)item1)->size > (*(ManifestFile **)item2)->size)
else if (file1.size > file2.size)
FUNCTION_TEST_RETURN(1);
// If size is the same then use name to generate a deterministic ordering (names must be unique)
FUNCTION_TEST_RETURN(strCmp((*(ManifestFile **)item1)->name, (*(ManifestFile **)item2)->name));
FUNCTION_TEST_RETURN(strCmp(file1.name, file2.name));
}
// Helper to generate the backup queues
@ -1479,20 +1489,21 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData)
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
const ManifestFile *file = manifestFile(manifest, fileIdx);
const ManifestFilePack *const filePack = manifestFilePackGet(manifest, fileIdx);
const ManifestFile file = manifestFileUnpack(filePack);
// If the file is a reference it should only be backed up if delta and not zero size
if (file->reference != NULL && (!jobData->delta || file->size == 0))
if (file.reference != NULL && (!jobData->delta || file.size == 0))
continue;
// Is pg_control in the backup?
if (strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)))
if (strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)))
pgControlFound = true;
// Files that must be copied from the primary are always put in queue 0 when backup from standby
if (jobData->backupStandby && backupProcessFilePrimary(jobData->standbyExp, file->name))
if (jobData->backupStandby && backupProcessFilePrimary(jobData->standbyExp, file.name))
{
lstAdd(*(List **)lstGet(jobData->queueList, 0), &file);
lstAdd(*(List **)lstGet(jobData->queueList, 0), &filePack);
}
// Else find the correct queue by matching the file to a target
else
@ -1504,7 +1515,7 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData)
{
CHECK(AssertError, targetIdx < strLstSize(targetList), "backup target not found");
if (strBeginsWith(file->name, strLstGet(targetList, targetIdx)))
if (strBeginsWith(file.name, strLstGet(targetList, targetIdx)))
break;
targetIdx++;
@ -1512,11 +1523,11 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData)
while (1);
// Add file to queue
lstAdd(*(List **)lstGet(jobData->queueList, targetIdx + queueOffset), &file);
lstAdd(*(List **)lstGet(jobData->queueList, targetIdx + queueOffset), &filePack);
}
// Add size to total
result += file->size;
result += file.size;
// Increment total files
fileTotal++;
@ -1600,21 +1611,21 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
if (!lstEmpty(queue))
{
const ManifestFile *file = *(ManifestFile **)lstGet(queue, 0);
const ManifestFile file = manifestFileUnpack(*(ManifestFilePack **)lstGet(queue, 0));
// Create backup job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE);
PackWrite *const param = protocolCommandParam(command);
pckWriteStrP(param, manifestPathPg(file->name));
pckWriteBoolP(param, !strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)));
pckWriteU64P(param, file->size);
pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file->name));
pckWriteStrP(param, file->checksumSha1[0] != 0 ? STR(file->checksumSha1) : NULL);
pckWriteBoolP(param, file->checksumPage);
pckWriteStrP(param, manifestPathPg(file.name));
pckWriteBoolP(param, !strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)));
pckWriteU64P(param, file.size);
pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file.name));
pckWriteStrP(param, file.checksumSha1[0] != 0 ? STR(file.checksumSha1) : NULL);
pckWriteBoolP(param, file.checksumPage);
pckWriteU64P(param, jobData->lsnStart);
pckWriteStrP(param, file->name);
pckWriteBoolP(param, file->reference != NULL);
pckWriteStrP(param, file.name);
pckWriteBoolP(param, file.reference != NULL);
pckWriteU32P(param, jobData->compressType);
pckWriteI32P(param, jobData->compressLevel);
pckWriteStrP(param, jobData->backupLabel);
@ -1628,7 +1639,7 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
// Assign job to result
MEM_CONTEXT_PRIOR_BEGIN()
{
result = protocolParallelJobNew(VARSTR(file->name), command);
result = protocolParallelJobNew(VARSTR(file.name), command);
}
MEM_CONTEXT_PRIOR_END();
@ -1774,7 +1785,7 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart
backupStandby && protocolParallelJobProcessId(job) > 1 ? backupData->hostStandby : backupData->hostPrimary,
storagePathP(
protocolParallelJobProcessId(job) > 1 ? storagePgIdx(pgIdx) : backupData->storagePrimary,
manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job)))->name)), fileRemove, job,
manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job))).name)), fileRemove, job,
sizeTotal, &sizeProgress);
}
@ -1814,22 +1825,22 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
const ManifestFile *const file = manifestFile(manifest, fileIdx);
const ManifestFile file = manifestFile(manifest, fileIdx);
// If the file has a reference, then it was not copied since it can be retrieved from the referenced backup. However,
// if hardlinking is enabled the link will need to be created.
if (file->reference != NULL)
if (file.reference != NULL)
{
// If hardlinking is enabled then create a hardlink for files that have not changed since the last backup
if (hardLink)
{
LOG_DETAIL_FMT("hardlink %s to %s", strZ(file->name), strZ(file->reference));
LOG_DETAIL_FMT("hardlink %s to %s", strZ(file.name), strZ(file.reference));
const String *const linkName = storagePathP(
storageRepo(), strNewFmt("%s/%s%s", strZ(backupPathExp), strZ(file->name), compressExt));
const String *const linkDestination = storagePathP(
storageRepo(), strNewFmt("%s/%s%s", strZ(backupPathExp), strZ(file.name), compressExt));
const String *const linkDestination = storagePathP(
storageRepo(),
strNewFmt(STORAGE_REPO_BACKUP "/%s/%s%s", strZ(file->reference), strZ(file->name), compressExt));
strNewFmt(STORAGE_REPO_BACKUP "/%s/%s%s", strZ(file.reference), strZ(file.name), compressExt));
THROW_ON_SYS_ERROR_FMT(
link(strZ(linkDestination), strZ(linkName)) == -1, FileOpenError,
@ -1838,7 +1849,7 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart
// Else log the reference. With delta, it is possible that references may have been removed if a file needed to be
// recopied.
else
LOG_DETAIL_FMT("reference %s to %s", strZ(file->name), strZ(file->reference));
LOG_DETAIL_FMT("reference %s to %s", strZ(file.name), strZ(file.reference));
}
}
@ -1974,7 +1985,7 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif
memcpy(file.checksumSha1, strZ(strSubN(archiveFile, 25, 40)), HASH_TYPE_SHA1_SIZE_HEX + 1);
manifestFileAdd(manifest, &file);
manifestFileAdd(manifest, file);
}
}
MEM_CONTEXT_TEMP_END();

6
src/command/info/info.c
View File

@ -533,10 +533,10 @@ backupListAdd(
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(repoData->manifest); fileIdx++)
{
const ManifestFile *file = manifestFile(repoData->manifest, fileIdx);
const ManifestFile file = manifestFile(repoData->manifest, fileIdx);
if (file->checksumPageError)
varLstAdd(checksumPageErrorList, varNewStr(manifestPathPg(file->name)));
if (file.checksumPageError)
varLstAdd(checksumPageErrorList, varNewStr(manifestPathPg(file.name)));
}
if (!varLstEmpty(checksumPageErrorList))

112
src/command/restore/restore.c
View File

@ -533,22 +533,22 @@ restoreManifestMap(Manifest *manifest)
if (target.name == NULL)
{
// Is the specified link a file or a path? Error if they both match.
const ManifestPath *const path = manifestPathFindDefault(manifest, manifestName, NULL);
const ManifestFile *const file = manifestFileFindDefault(manifest, manifestName, NULL);
const bool pathExists = manifestPathFindDefault(manifest, manifestName, NULL) != NULL;
const bool fileExists = manifestFileExists(manifest, manifestName);
CHECK(FormatError, path == NULL || file == NULL, "link may not be both file and path");
CHECK(FormatError, !pathExists || !fileExists, "link may not be both file and path");
target = (ManifestTarget){.name = manifestName, .path = linkPath, .type = manifestTargetTypeLink};
// If a file
if (file != NULL)
if (fileExists)
{
// File needs to be set so the file/path is updated later but set it to something invalid just in case it
// it does not get updated due to a regression
target.file = DOT_STR;
}
// Else error if not a path
else if (path == NULL)
else if (!pathExists)
{
THROW_FMT(
LinkMapError,
@ -666,20 +666,20 @@ restoreManifestOwnerReplace(const String *const owner, const String *const owner
}
// Helper to get list of owners from a file/link/path list
#define RESTORE_MANIFEST_OWNER_GET(type) \
#define RESTORE_MANIFEST_OWNER_GET(type, deref) \
for (unsigned int itemIdx = 0; itemIdx < manifest##type##Total(manifest); itemIdx++) \
{ \
Manifest##type *item = (Manifest##type *)manifest##type(manifest, itemIdx); \
Manifest##type item = deref manifest##type(manifest, itemIdx); \
\
if (item->user == NULL) \
if (item.user == NULL) \
userNull = true; \
else \
strLstAddIfMissing(userList, item->user); \
strLstAddIfMissing(userList, item.user); \
\
if (item->group == NULL) \
if (item.group == NULL) \
groupNull = true; \
else \
strLstAddIfMissing(groupList, item->group); \
strLstAddIfMissing(groupList, item.group); \
}
// Helper to warn when an owner is missing and must be remapped
@ -719,9 +719,9 @@ restoreManifestOwner(const Manifest *const manifest, const String **const rootRe
bool groupNull = false;
StringList *groupList = strLstNew();
RESTORE_MANIFEST_OWNER_GET(File);
RESTORE_MANIFEST_OWNER_GET(Link);
RESTORE_MANIFEST_OWNER_GET(Path);
RESTORE_MANIFEST_OWNER_GET(File, );
RESTORE_MANIFEST_OWNER_GET(Link, *(ManifestLink *));
RESTORE_MANIFEST_OWNER_GET(Path, *(ManifestPath *));
// Update users and groups in the manifest (this can only be done as root)
// -------------------------------------------------------------------------------------------------------------------------
@ -918,14 +918,15 @@ restoreCleanInfoListCallback(void *data, const StorageInfo *info)
{
case storageTypeFile:
{
const ManifestFile *manifestFile = manifestFileFindDefault(cleanData->manifest, manifestName, NULL);
if (manifestFile != NULL && manifestLinkFindDefault(cleanData->manifest, manifestName, NULL) == NULL)
if (manifestFileExists(cleanData->manifest, manifestName) &&
manifestLinkFindDefault(cleanData->manifest, manifestName, NULL) == NULL)
{
const ManifestFile manifestFile = manifestFileFind(cleanData->manifest, manifestName);
restoreCleanOwnership(
pgPath, manifestFile->user, cleanData->rootReplaceUser, manifestFile->group, cleanData->rootReplaceGroup,
pgPath, manifestFile.user, cleanData->rootReplaceUser, manifestFile.group, cleanData->rootReplaceGroup,
info->userId, info->groupId, false);
restoreCleanMode(pgPath, manifestFile->mode, info);
restoreCleanMode(pgPath, manifestFile.mode, info);
}
else
{
@ -1133,7 +1134,7 @@ restoreCleanBuild(const Manifest *const manifest, const String *const rootReplac
// Skip the tablespace_map file when present so PostgreSQL does not rewrite links in pg_tblspc. The tablespace links will be
// created after paths are cleaned.
if (manifestFileFindDefault(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_TABLESPACEMAP), NULL) != NULL &&
if (manifestFileExists(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_TABLESPACEMAP)) &&
manifestData(manifest)->pgVersion >= PG_VERSION_TABLESPACE_MAP)
{
LOG_DETAIL_FMT("skip '" PG_FILE_TABLESPACEMAP "' -- tablespace links will be created based on mappings");
@ -1142,7 +1143,7 @@ restoreCleanBuild(const Manifest *const manifest, const String *const rootReplac
// Skip postgresql.auto.conf if preserve is set and the PostgreSQL version supports recovery GUCs
if (manifestData(manifest)->pgVersion >= PG_VERSION_RECOVERY_GUC && cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_PRESERVE &&
manifestFileFindDefault(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF), NULL) != NULL)
manifestFileExists(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)))
{
LOG_DETAIL_FMT("skip '" PG_FILE_POSTGRESQLAUTOCONF "' -- recovery type is preserve");
manifestFileRemove(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF));
@ -1341,11 +1342,11 @@ restoreSelectiveExpression(Manifest *manifest)
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
const ManifestFile *file = manifestFile(manifest, fileIdx);
const String *const fileName = manifestFileNameGet(manifest, fileIdx);
if (regExpMatch(baseRegExp, file->name) || regExpMatch(tablespaceRegExp, file->name))
if (regExpMatch(baseRegExp, fileName) || regExpMatch(tablespaceRegExp, fileName))
{
String *dbId = strBase(strPath(file->name));
String *dbId = strBase(strPath(fileName));
// In the highly unlikely event that a system database was somehow added after the backup began, it will only be
// found in the file list and not the manifest db section, so add it to the system database list
@ -1907,14 +1908,18 @@ restoreProcessQueueComparator(const void *item1, const void *item2)
ASSERT(item1 != NULL);
ASSERT(item2 != NULL);
// Unpack files
ManifestFile file1 = manifestFileUnpack(*(const ManifestFilePack **)item1);
ManifestFile file2 = manifestFileUnpack(*(const ManifestFilePack **)item2);
// If the size differs then that's enough to determine order
if ((*(ManifestFile **)item1)->size < (*(ManifestFile **)item2)->size)
if (file1.size < file2.size)
FUNCTION_TEST_RETURN(-1);
else if ((*(ManifestFile **)item1)->size > (*(ManifestFile **)item2)->size)
else if (file1.size > file2.size)
FUNCTION_TEST_RETURN(1);
// If size is the same then use name to generate a deterministic ordering (names must be unique)
FUNCTION_TEST_RETURN(strCmp((*(ManifestFile **)item1)->name, (*(ManifestFile **)item2)->name));
FUNCTION_TEST_RETURN(strCmp(file1.name, file2.name));
}
static uint64_t
@ -1960,7 +1965,8 @@ restoreProcessQueue(Manifest *manifest, List **queueList)
// Now put all files into the processing queues
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
const ManifestFile *file = manifestFile(manifest, fileIdx);
const ManifestFilePack *const filePack = manifestFilePackGet(manifest, fileIdx);
const ManifestFile file = manifestFileUnpack(filePack);
// Find the target that contains this file
unsigned int targetIdx = 0;
@ -1970,7 +1976,7 @@ restoreProcessQueue(Manifest *manifest, List **queueList)
// A target should always be found
CHECK(FormatError, targetIdx < strLstSize(targetList), "backup target not found");
if (strBeginsWith(file->name, strLstGet(targetList, targetIdx)))
if (strBeginsWith(file.name, strLstGet(targetList, targetIdx)))
break;
targetIdx++;
@ -1978,10 +1984,10 @@ restoreProcessQueue(Manifest *manifest, List **queueList)
while (1);
// Add file to queue
lstAdd(*(List **)lstGet(*queueList, targetIdx), &file);
lstAdd(*(List **)lstGet(*queueList, targetIdx), &filePack);
// Add size to total
result += file->size;
result += file.size;
}
// Sort the queues
@ -2052,8 +2058,8 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
{
MEM_CONTEXT_TEMP_BEGIN()
{
const ManifestFile *file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
bool zeroed = restoreFileZeroed(file->name, zeroExp);
const ManifestFile file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
bool zeroed = restoreFileZeroed(file.name, zeroExp);
bool copy = pckReadBoolP(protocolParallelJobResult(job));
String *log = strCatZ(strNew(), "restore");
@ -2063,7 +2069,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
strCatZ(log, " zeroed");
// Add filename
strCatFmt(log, " file %s", strZ(restoreFilePgPath(manifest, file->name)));
strCatFmt(log, " file %s", strZ(restoreFilePgPath(manifest, file.name)));
// If not copied and not zeroed add details to explain why it was not copied
if (!copy && !zeroed)
@ -2074,8 +2080,8 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
if (cfgOptionBool(cfgOptForce))
{
strCatFmt(
log, "exists and matches size %" PRIu64 " and modification time %" PRIu64, file->size,
(uint64_t)file->timestamp);
log, "exists and matches size %" PRIu64 " and modification time %" PRIu64, file.size,
(uint64_t)file.timestamp);
}
// Else a checksum delta or file is zero-length
else
@ -2083,7 +2089,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
strCatZ(log, "exists and ");
// No need to copy zero-length files
if (file->size == 0)
if (file.size == 0)
{
strCatZ(log, "is zero size");
}
@ -2094,12 +2100,12 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
}
// Add size and percent complete
sizeRestored += file->size;
strCatFmt(log, " (%s, %" PRIu64 "%%)", strZ(strSizeFormat(file->size)), sizeRestored * 100 / sizeTotal);
sizeRestored += file.size;
strCatFmt(log, " (%s, %" PRIu64 "%%)", strZ(strSizeFormat(file.size)), sizeRestored * 100 / sizeTotal);
// If not zero-length add the checksum
if (file->size != 0 && !zeroed)
strCatFmt(log, " checksum %s", file->checksumSha1);
if (file.size != 0 && !zeroed)
strCatFmt(log, " checksum %s", file.checksumSha1);
LOG_DETAIL_PID(protocolParallelJobProcessId(job), strZ(log));
}
@ -2178,24 +2184,24 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
if (!lstEmpty(queue))
{
const ManifestFile *file = *(ManifestFile **)lstGet(queue, 0);
const ManifestFile file = manifestFileUnpack(*(ManifestFilePack **)lstGet(queue, 0));
// Create restore job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
PackWrite *const param = protocolCommandParam(command);
pckWriteStrP(param, file->name);
pckWriteStrP(param, file.name);
pckWriteU32P(param, jobData->repoIdx);
pckWriteStrP(param, file->reference != NULL ? file->reference : manifestData(jobData->manifest)->backupLabel);
pckWriteStrP(param, file.reference != NULL ? file.reference : manifestData(jobData->manifest)->backupLabel);
pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType);
pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file->name));
pckWriteStrP(param, STR(file->checksumSha1));
pckWriteBoolP(param, restoreFileZeroed(file->name, jobData->zeroExp));
pckWriteU64P(param, file->size);
pckWriteTimeP(param, file->timestamp);
pckWriteModeP(param, file->mode);
pckWriteStrP(param, restoreManifestOwnerReplace(file->user, jobData->rootReplaceUser));
pckWriteStrP(param, restoreManifestOwnerReplace(file->group, jobData->rootReplaceGroup));
pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file.name));
pckWriteStrP(param, STR(file.checksumSha1));
pckWriteBoolP(param, restoreFileZeroed(file.name, jobData->zeroExp));
pckWriteU64P(param, file.size);
pckWriteTimeP(param, file.timestamp);
pckWriteModeP(param, file.mode);
pckWriteStrP(param, restoreManifestOwnerReplace(file.user, jobData->rootReplaceUser));
pckWriteStrP(param, restoreManifestOwnerReplace(file.group, jobData->rootReplaceGroup));
pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart);
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta));
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce));
@ -2207,7 +2213,7 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
// Assign job to result
MEM_CONTEXT_PRIOR_BEGIN()
{
result = protocolParallelJobNew(VARSTR(file->name), command);
result = protocolParallelJobNew(VARSTR(file.name), command);
}
MEM_CONTEXT_PRIOR_END();

18
src/command/verify/verify.c
View File

@ -902,7 +902,7 @@ verifyBackup(void *data)
{
do
{
const ManifestFile *fileData = manifestFile(jobData->manifest, jobData->manifestFileIdx);
const ManifestFile fileData = manifestFile(jobData->manifest, jobData->manifestFileIdx);
String *filePathName = NULL;
@ -910,16 +910,16 @@ verifyBackup(void *data)
backupResult->totalFileVerify++;
// Check if the file is referenced in a prior backup
if (fileData->reference != NULL)
if (fileData.reference != NULL)
{
// If the prior backup is not in the result list, then that backup was never processed (likely due to the --set
// option) so verify the file
unsigned int backupPriorIdx = lstFindIdx(jobData->backupResultList, &fileData->reference);
unsigned int backupPriorIdx = lstFindIdx(jobData->backupResultList, &fileData.reference);
if (backupPriorIdx == LIST_NOT_FOUND)
{
filePathName = strNewFmt(
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name),
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData.reference), strZ(fileData.name),
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
}
// Else the backup this file references has a result so check the processing state for the referenced backup
@ -931,14 +931,14 @@ verifyBackup(void *data)
if (!backupResultPrior->fileVerifyComplete)
{
filePathName = strNewFmt(
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name),
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData.reference), strZ(fileData.name),
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
}
// Else skip verification
else
{
String *priorFile = strNewFmt(
"%s/%s%s", strZ(fileData->reference), strZ(fileData->name),
"%s/%s%s", strZ(fileData.reference), strZ(fileData.name),
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
unsigned int backupPriorInvalidIdx = lstFindIdx(backupResultPrior->invalidFileList, &priorFile);
@ -965,7 +965,7 @@ verifyBackup(void *data)
else
{
filePathName = strNewFmt(
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(backupResult->backupLabel), strZ(fileData->name),
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(backupResult->backupLabel), strZ(fileData.name),
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
}
@ -978,8 +978,8 @@ verifyBackup(void *data)
pckWriteStrP(param, filePathName);
// If the checksum is not present in the manifest, it will be calculated by manifest load
pckWriteStrP(param, STR(fileData->checksumSha1));
pckWriteU64P(param, fileData->size);
pckWriteStrP(param, STR(fileData.checksumSha1));
pckWriteU64P(param, fileData.size);
pckWriteStrP(param, jobData->backupCipherPass);
// Assign job to result (prepend backup label being processed to the key since some files are in a prior backup)

77
src/common/type/convert.c
View File

@ -482,3 +482,80 @@ cvtZToUInt64(const char *value)
FUNCTION_TEST_RETURN(cvtZToUInt64Base(value, 10));
}
/**********************************************************************************************************************************/
void
cvtUInt64ToVarInt128(uint64_t value, uint8_t *const buffer, size_t *const bufferPos, const size_t bufferSize)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(UINT64, value);
FUNCTION_TEST_PARAM_P(VOID, buffer);
FUNCTION_TEST_PARAM(UINT64, bufferSize);
FUNCTION_TEST_END();
ASSERT(buffer != NULL);
ASSERT(bufferPos != NULL);
ASSERT(bufferSize > *bufferPos);
// Keep encoding bytes while the remaining value is greater than 7 bits
while (value >= 0x80)
{
// Encode the lower order 7 bits, adding the continuation bit to indicate there is more data
buffer[*bufferPos] = (unsigned char)value | 0x80;
// Shift the value to remove bits that have been encoded
value >>= 7;
// Keep track of size so we know how many bytes to write out
(*bufferPos)++;
// Make sure the buffer won't overflow
if (*bufferPos >= bufferSize)
THROW(AssertError, "buffer overflow");
}
// Encode the last 7 bits of value
buffer[*bufferPos] = (unsigned char)value;
(*bufferPos)++;
FUNCTION_TEST_RETURN_VOID();
}
uint64_t
cvtUInt64FromVarInt128(const uint8_t *const value, size_t *const valuePos)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, value);
FUNCTION_TEST_PARAM_P(UINT64, valuePos);
FUNCTION_TEST_END();
ASSERT(value != NULL);
ASSERT(valuePos != NULL);
// Decode all bytes
uint64_t result = 0;
uint8_t byte;
for (unsigned int valueIdx = 0; valueIdx < CVT_VARINT128_BUFFER_SIZE; valueIdx++)
{
// Get the next encoded byte
byte = value[*valuePos];
// Shift the lower order 7 encoded bits into the uint64 in reverse order
result |= (uint64_t)(byte & 0x7f) << (7 * valueIdx);
// Increment value position to indicate that the byte has been processed
(*valuePos)++;
// Done if the high order bit is not set to indicate more data
if (byte < 0x80)
break;
}
// By this point all bytes should have been read so error if this is not the case. This could be due to a coding error or
// corrupton in the data stream.
if (byte >= 0x80)
THROW(FormatError, "unterminated varint-128 integer");
FUNCTION_TEST_RETURN(result);
}

5
src/common/type/convert.h
View File

@ -15,6 +15,7 @@ Required buffer sizes
***********************************************************************************************************************************/
#define CVT_BOOL_BUFFER_SIZE 6
#define CVT_BASE10_BUFFER_SIZE 64
#define CVT_VARINT128_BUFFER_SIZE 10
/***********************************************************************************************************************************
Functions
@ -85,6 +86,10 @@ size_t cvtUInt64ToZ(uint64_t value, char *buffer, size_t bufferSize);
uint64_t cvtZToUInt64(const char *value);
uint64_t cvtZToUInt64Base(const char *value, int base);
// Convert uint64 to base-128 varint and vice versa
void cvtUInt64ToVarInt128(uint64_t value, uint8_t *buffer, size_t *bufferPos, size_t bufferSize);
uint64_t cvtUInt64FromVarInt128(const uint8_t *value, size_t *valuePos);
// Convert boolean to zero-terminated string. Use cvtBoolToConstZ() whenever possible since it is more efficient.
size_t cvtBoolToZ(bool value, char *buffer, size_t bufferSize);
const char *cvtBoolToConstZ(bool value);

29
src/common/type/pack.c
View File

@ -104,11 +104,6 @@ Array and object types:
#include "common/type/convert.h"
#include "common/type/pack.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define PACK_UINT64_SIZE_MAX 10
/***********************************************************************************************************************************
Map PackType types to the types that will be written into the pack. This hides the details of the type IDs from the user and allows
the IDs used in the pack to differ from the IDs the user sees.
@ -495,7 +490,7 @@ pckReadU64Internal(PackRead *this)
uint8_t byte;
// Convert bytes from varint-128 encoding to a uint64
for (unsigned int bufferIdx = 0; bufferIdx < PACK_UINT64_SIZE_MAX; bufferIdx++)
for (unsigned int bufferIdx = 0; bufferIdx < CVT_VARINT128_BUFFER_SIZE; bufferIdx++)
{
// Get the next encoded byte
pckReadBuffer(this, 1);
@ -1409,7 +1404,7 @@ pckWriteBuffer(PackWrite *this, const Buffer *buffer)
Pack an unsigned 64-bit integer to base-128 varint encoding
***********************************************************************************************************************************/
static void
pckWriteU64Internal(PackWrite *this, uint64_t value)
pckWriteU64Internal(PackWrite *const this, const uint64_t value)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PACK_WRITE, this);
@ -1418,27 +1413,13 @@ pckWriteU64Internal(PackWrite *this, uint64_t value)
ASSERT(this != NULL);
unsigned char buffer[PACK_UINT64_SIZE_MAX];
unsigned char buffer[CVT_VARINT128_BUFFER_SIZE];
size_t size = 0;
// Convert uint64 to varint-128 encoding. Keep writing out bytes while the remaining value is greater than 7 bits.
while (value >= 0x80)
{
// Encode the lower order 7 bits, adding the continuation bit to indicate there is more data
buffer[size] = (unsigned char)value | 0x80;
// Shift the value to remove bits that have been encoded
value >>= 7;
// Keep track of size so we know how many bytes to write out
size++;
}
// Encode the last 7 bits of value
buffer[size] = (unsigned char)value;
cvtUInt64ToVarInt128(value, buffer, &size, sizeof(buffer));
// Write encoded bytes to the buffer
pckWriteBuffer(this, BUF(buffer, size + 1));
pckWriteBuffer(this, BUF(buffer, size));
FUNCTION_TEST_RETURN_VOID();
}

16
src/info/infoBackup.c
View File

@ -370,22 +370,22 @@ infoBackupDataAdd(const InfoBackup *this, const Manifest *manifest)
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
const ManifestFile *file = manifestFile(manifest, fileIdx);
const ManifestFile file = manifestFile(manifest, fileIdx);
backupSize += file->size;
backupRepoSize += file->sizeRepo > 0 ? file->sizeRepo : file->size;
backupSize += file.size;
backupRepoSize += file.sizeRepo > 0 ? file.sizeRepo : file.size;
// If a reference to a file exists, then it is in a previous backup and the delta calculation was already done
if (file->reference != NULL)
strLstAddIfMissing(referenceList, file->reference);
if (file.reference != NULL)
strLstAddIfMissing(referenceList, file.reference);
else
{
backupSizeDelta += file->size;
backupRepoSizeDelta += file->sizeRepo > 0 ? file->sizeRepo : file->size;
backupSizeDelta += file.size;
backupRepoSizeDelta += file.sizeRepo > 0 ? file.sizeRepo : file.size;
}
// Is there an error in the file?
if (file->checksumPageError)
if (file.checksumPageError)
backupError = true;
}

434
src/info/manifest.c
View File

@ -198,59 +198,208 @@ manifestDbAdd(Manifest *this, const ManifestDb *db)
FUNCTION_TEST_RETURN_VOID();
}
// Base time used as a delta to reduce the size of packed timestamps. This will be set on the first call to manifestFilePack().
static time_t manifestPackBaseTime = -1;
// Flags used to reduce the size of packed data. They should be ordered from most to least likely and can be reordered at will.
typedef enum
{
manifestFilePackFlagReference,
manifestFilePackFlagChecksumPage,
manifestFilePackFlagChecksumPageError,
manifestFilePackFlagChecksumPageErrorList,
} ManifestFilePackFlag;
// Pack file into a compact format to save memory
static ManifestFilePack *
manifestFilePack(const ManifestFile *const file)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST_FILE, file);
FUNCTION_TEST_END();
uint8_t buffer[512];
size_t bufferPos = 0;
// Flags
uint64_t flag = 0;
if (file->checksumPage)
flag |= 1 << manifestFilePackFlagChecksumPage;
if (file->checksumPageError)
flag |= 1 << manifestFilePackFlagChecksumPageError;
if (file->checksumPageErrorList != NULL)
flag |= 1 << manifestFilePackFlagChecksumPageErrorList;
if (file->reference != NULL)
flag |= 1 << manifestFilePackFlagReference;
cvtUInt64ToVarInt128(flag, buffer, &bufferPos, sizeof(buffer));
// Size
cvtUInt64ToVarInt128(file->size, buffer, &bufferPos, sizeof(buffer));
// Use the first timestamp that appears as the base for all other timestamps. Ideally we would like a timestamp as close to the
// middle as possible but it doesn't seem worth doing the calculation.
if (manifestPackBaseTime == -1)
manifestPackBaseTime = file->timestamp;
// Timestamp
cvtUInt64ToVarInt128(cvtInt64ToZigZag(manifestPackBaseTime - file->timestamp), buffer, &bufferPos, sizeof(buffer));
// SHA1 checksum
strcpy((char *)buffer + bufferPos, file->checksumSha1);
bufferPos += HASH_TYPE_SHA1_SIZE_HEX + 1;
// Reference
if (file->reference != NULL)
cvtUInt64ToVarInt128((uintptr_t)file->reference, buffer, &bufferPos, sizeof(buffer));
// Mode
cvtUInt64ToVarInt128(file->mode, buffer, &bufferPos, sizeof(buffer));
// User/group
cvtUInt64ToVarInt128((uintptr_t)file->user, buffer, &bufferPos, sizeof(buffer));
cvtUInt64ToVarInt128((uintptr_t)file->group, buffer, &bufferPos, sizeof(buffer));
// Repo size
cvtUInt64ToVarInt128(file->sizeRepo, buffer, &bufferPos, sizeof(buffer));
// Allocate memory for the file pack
uint8_t *const result = memNew(
sizeof(StringPub) + strSize(file->name) + 1 + bufferPos + (file->checksumPageErrorList != NULL ?
sizeof(StringPub) + strSize(file->checksumPageErrorList) + 1 : 0));
// Create string object for the file name
*(StringPub *)result = (StringPub){.size = (unsigned int)strSize(file->name), .buffer = (char *)result + sizeof(StringPub)};
size_t resultPos = sizeof(StringPub);
memcpy(result + resultPos, (uint8_t *)strZ(file->name), strSize(file->name) + 1);
resultPos += strSize(file->name) + 1;
// Copy pack data
memcpy(result + resultPos, buffer, bufferPos);
// Create string object for the checksum error list
if (file->checksumPageErrorList != NULL)
{
resultPos += bufferPos;
*(StringPub *)(result + resultPos) = (StringPub)
{.size = (unsigned int)strSize(file->checksumPageErrorList), .buffer = (char *)result + resultPos + sizeof(StringPub)};
resultPos += sizeof(StringPub);
memcpy(result + resultPos, (uint8_t *)strZ(file->checksumPageErrorList), strSize(file->checksumPageErrorList) + 1);
}
FUNCTION_TEST_RETURN((ManifestFilePack *)result);
}
ManifestFile
manifestFileUnpack(const ManifestFilePack *const filePack)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, filePack);
FUNCTION_TEST_END();
ASSERT(filePack != NULL);
ASSERT(manifestPackBaseTime != -1);
ManifestFile result = {0};
size_t bufferPos = 0;
// Name
result.name = (const String *)filePack;
bufferPos += sizeof(StringPub) + strSize(result.name) + 1;
// Flags
const uint64_t flag = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// Size
result.size = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// Timestamp
result.timestamp =
manifestPackBaseTime - (time_t)cvtInt64FromZigZag(cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos));
// Checksum page
result.checksumPage = flag & (1 << manifestFilePackFlagChecksumPage) ? true : false;
// SHA1 checksum
memcpy(result.checksumSha1, (const uint8_t *)filePack + bufferPos, HASH_TYPE_SHA1_SIZE_HEX + 1);
bufferPos += HASH_TYPE_SHA1_SIZE_HEX + 1;
// Reference
if (flag & (1 << manifestFilePackFlagReference))
result.reference = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// Mode
result.mode = (mode_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// User/group
result.user = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
result.group = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// Repo size
result.sizeRepo = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
// Checksum page error
result.checksumPageError = flag & (1 << manifestFilePackFlagChecksumPageError) ? true : false;
if (flag & (1 << manifestFilePackFlagChecksumPageErrorList))
result.checksumPageErrorList = (const String *)((const uint8_t *)filePack + bufferPos);
FUNCTION_TEST_RETURN(result);
}
void
manifestFileAdd(Manifest *this, const ManifestFile *file)
manifestFileAdd(Manifest *this, ManifestFile file)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST, this);
FUNCTION_TEST_PARAM(VOID, file);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(file.name != NULL);
file.user = manifestOwnerCache(this, file.user);
file.group = manifestOwnerCache(this, file.group);
if (file.reference != NULL)
file.reference = strLstAddIfMissing(this->referenceList, file.reference);
MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
{
const ManifestFilePack *const filePack = manifestFilePack(&file);
lstAdd(this->pub.fileList, &filePack);
}
MEM_CONTEXT_END();
FUNCTION_TEST_RETURN_VOID();
}
// Update file pack by creating a new one and then freeing the old one
static void
manifestFilePackUpdate(Manifest *const this, ManifestFilePack **const filePack, const ManifestFile *const file)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST, this);
FUNCTION_TEST_PARAM_P(VOID, filePack);
FUNCTION_TEST_PARAM(MANIFEST_FILE, file);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(filePack != NULL);
ASSERT(file != NULL);
ASSERT(file->name != NULL);
MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
{
ManifestFile fileAdd =
{
.checksumPage = file->checksumPage,
.checksumPageError = file->checksumPageError,
.checksumPageErrorList = varLstDup(file->checksumPageErrorList),
.group = manifestOwnerCache(this, file->group),
.mode = file->mode,
.name = strDup(file->name),
.size = file->size,
.sizeRepo = file->sizeRepo,
.timestamp = file->timestamp,
.user = manifestOwnerCache(this, file->user),
};
memcpy(fileAdd.checksumSha1, file->checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
if (file->reference != NULL)
{
// Search for the reference in the list
for (unsigned int referenceIdx = 0; referenceIdx < strLstSize(this->referenceList); referenceIdx++)
{
const String *found = strLstGet(this->referenceList, referenceIdx);
if (strEq(file->reference, found))
{
fileAdd.reference = found;
break;
}
}
// If not found then add it
if (fileAdd.reference == NULL)
{
strLstAdd(this->referenceList, file->reference);
fileAdd.reference = strLstGet(this->referenceList, strLstSize(this->referenceList) - 1);
}
}
lstAdd(this->pub.fileList, &fileAdd);
ManifestFilePack *const filePackOld = *filePack;
*filePack = manifestFilePack(file);
memFree(filePackOld);
}
MEM_CONTEXT_END();
@ -364,7 +513,7 @@ manifestNewInternal(void)
{
.memContext = memContextCurrent(),
.dbList = lstNewP(sizeof(ManifestDb), .comparator = lstComparatorStr),
.fileList = lstNewP(sizeof(ManifestFile), .comparator = lstComparatorStr),
.fileList = lstNewP(sizeof(ManifestFilePack *), .comparator = lstComparatorStr),
.linkList = lstNewP(sizeof(ManifestLink), .comparator = lstComparatorStr),
.pathList = lstNewP(sizeof(ManifestPath), .comparator = lstComparatorStr),
.targetList = lstNewP(sizeof(ManifestTarget), .comparator = lstComparatorStr),
@ -843,7 +992,7 @@ manifestBuildCallback(void *data, const StorageInfo *info)
!strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL);
}
manifestFileAdd(buildData.manifest, &file);
manifestFileAdd(buildData.manifest, file);
break;
}
@ -1163,12 +1312,12 @@ manifestNewBuild(
while (fileIdx < manifestFileTotal(this))
{
// If this file looks like a relation. Note that this never matches on _init forks.
const ManifestFile *file = manifestFile(this, fileIdx);
const String *const filePathName = manifestFileNameGet(this, fileIdx);
if (regExpMatch(relationExp, file->name))
if (regExpMatch(relationExp, filePathName))
{
// Get the filename (without path)
const char *fileName = strBaseZ(file->name);
const char *fileName = strBaseZ(filePathName);
size_t fileNameSize = strlen(fileName);
// Strip off the numeric part of the relation
@ -1193,8 +1342,8 @@ manifestNewBuild(
{
// Determine if the relation is unlogged
String *relationInit = strNewFmt(
"%.*s%s_init", (int)(strSize(file->name) - fileNameSize), strZ(file->name), relationFileId);
lastRelationFileIdUnlogged = manifestFileFindDefault(this, relationInit, NULL) != NULL;
"%.*s%s_init", (int)(strSize(filePathName) - fileNameSize), strZ(filePathName), relationFileId);
lastRelationFileIdUnlogged = manifestFileExists(this, relationInit);
strFree(relationInit);
// Save the file id so we don't need to do the lookup next time if it doesn't change
@ -1204,7 +1353,7 @@ manifestNewBuild(
// If relation is unlogged then remove it
if (lastRelationFileIdUnlogged)
{
manifestFileRemove(this, file->name);
manifestFileRemove(this, filePathName);
continue;
}
}
@ -1262,13 +1411,13 @@ manifestBuildValidate(Manifest *this, bool delta, time_t copyStart, CompressType
{
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
{
const ManifestFile *file = manifestFile(this, fileIdx);
const ManifestFile file = manifestFile(this, fileIdx);
// Check for timestamp in the future
if (file->timestamp > copyStart)
if (file.timestamp > copyStart)
{
LOG_WARN_FMT(
"file '%s' has timestamp in the future, enabling delta checksum", strZ(manifestPathPg(file->name)));
"file '%s' has timestamp in the future, enabling delta checksum", strZ(manifestPathPg(file.name)));
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
break;
@ -1337,29 +1486,30 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
{
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
{
const ManifestFile *file = manifestFile(this, fileIdx);
const ManifestFile *filePrior = manifestFileFindDefault(manifestPrior, file->name, NULL);
const ManifestFile file = manifestFile(this, fileIdx);
// If file was found in prior manifest then perform checks
if (filePrior != NULL)
if (manifestFileExists(manifestPrior, file.name))
{
const ManifestFile filePrior = manifestFileFind(manifestPrior, file.name);
// Check for timestamp earlier than the prior backup
if (file->timestamp < filePrior->timestamp)
if (file.timestamp < filePrior.timestamp)
{
LOG_WARN_FMT(
"file '%s' has timestamp earlier than prior backup, enabling delta checksum",
strZ(manifestPathPg(file->name)));
strZ(manifestPathPg(file.name)));
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
break;
}
// Check for size change with no timestamp change
if (file->size != filePrior->size && file->timestamp == filePrior->timestamp)
if (file.size != filePrior.size && file.timestamp == filePrior.timestamp)
{
LOG_WARN_FMT(
"file '%s' has same timestamp as prior but different size, enabling delta checksum",
strZ(manifestPathPg(file->name)));
strZ(manifestPathPg(file.name)));
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
break;
@ -1375,17 +1525,20 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
for (unsigned int fileIdx = 0; fileIdx < lstSize(this->pub.fileList); fileIdx++)
{
const ManifestFile *file = manifestFile(this, fileIdx);
const ManifestFile *filePrior = manifestFileFindDefault(manifestPrior, file->name, NULL);
const ManifestFile file = manifestFile(this, fileIdx);
// Check if prior file can be used
if (filePrior != NULL && file->size == filePrior->size &&
(delta || file->size == 0 || file->timestamp == filePrior->timestamp))
if (manifestFileExists(manifestPrior, file.name))
{
manifestFileUpdate(
this, file->name, file->size, filePrior->sizeRepo, filePrior->checksumSha1,
VARSTR(filePrior->reference != NULL ? filePrior->reference : manifestPrior->pub.data.backupLabel),
filePrior->checksumPage, filePrior->checksumPageError, filePrior->checksumPageErrorList);
const ManifestFile filePrior = manifestFileFind(manifestPrior, file.name);
if (file.size == filePrior.size && (delta || file.size == 0 || file.timestamp == filePrior.timestamp))
{
manifestFileUpdate(
this, file.name, file.size, filePrior.sizeRepo, filePrior.checksumSha1,
VARSTR(filePrior.reference != NULL ? filePrior.reference : manifestPrior->pub.data.backupLabel),
filePrior.checksumPage, filePrior.checksumPageError, filePrior.checksumPageErrorList);
}
}
}
}
@ -1618,7 +1771,7 @@ manifestLoadCallback(void *callbackData, const String *section, const String *ke
const Variant *checksumPageErrorList = kvGetDefault(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, NULL);
if (checksumPageErrorList != NULL)
file.checksumPageErrorList = varVarLst(checksumPageErrorList);
file.checksumPageErrorList = jsonFromVar(checksumPageErrorList);
}
if (kvKeyExists(fileKv, MANIFEST_KEY_GROUP_VAR))
@ -1640,7 +1793,7 @@ manifestLoadCallback(void *callbackData, const String *section, const String *ke
}
lstAdd(loadData->fileFoundList, &valueFound);
manifestFileAdd(manifest, &file);
manifestFileAdd(manifest, file);
}
MEM_CONTEXT_END();
}
@ -1926,17 +2079,20 @@ manifestNewLoad(IoRead *read)
// Process file defaults
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
{
ManifestFile *file = lstGet(this->pub.fileList, fileIdx);
ManifestFilePack **const filePack = lstGet(this->pub.fileList, fileIdx);
ManifestFile file = manifestFileUnpack(*filePack);
ManifestLoadFound *found = lstGet(loadData.fileFoundList, fileIdx);
if (!found->group)
file->group = manifestOwnerCache(this, manifestOwnerGet(loadData.fileGroupDefault));
file.group = manifestOwnerCache(this, manifestOwnerGet(loadData.fileGroupDefault));
if (!found->mode)
file->mode = loadData.fileModeDefault;
file.mode = loadData.fileModeDefault;
if (!found->user)
file->user = manifestOwnerCache(this, manifestOwnerGet(loadData.fileUserDefault));
file.user = manifestOwnerCache(this, manifestOwnerGet(loadData.fileUserDefault));
manifestFilePackUpdate(this, filePack, &file);
}
// Process link defaults
@ -2243,41 +2399,41 @@ manifestSaveCallback(void *callbackData, const String *sectionNext, InfoSave *in
{
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
const ManifestFile *file = manifestFile(manifest, fileIdx);
const ManifestFile file = manifestFile(manifest, fileIdx);
KeyValue *fileKv = kvNew();
// Save if the file size is not zero and the checksum exists. The checksum might not exist if this is a partial
// save performed during a backup.
if (file->size != 0 && file->checksumSha1[0] != 0)
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_VAR, VARSTRZ(file->checksumSha1));
if (file.size != 0 && file.checksumSha1[0] != 0)
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_VAR, VARSTRZ(file.checksumSha1));
if (file->checksumPage)
if (file.checksumPage)
{
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, VARBOOL(!file->checksumPageError));
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, VARBOOL(!file.checksumPageError));
if (file->checksumPageErrorList != NULL)
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, varNewVarLst(file->checksumPageErrorList));
if (file.checksumPageErrorList != NULL)
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, jsonToVar(file.checksumPageErrorList));
}
if (!varEq(manifestOwnerVar(file->group), saveData->groupDefault))
kvPut(fileKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(file->group));
if (!varEq(manifestOwnerVar(file.group), saveData->groupDefault))
kvPut(fileKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(file.group));
if (file->mode != saveData->fileModeDefault)
kvPut(fileKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", file->mode)));
if (file.mode != saveData->fileModeDefault)
kvPut(fileKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", file.mode)));
if (file->reference != NULL)
kvPut(fileKv, MANIFEST_KEY_REFERENCE_VAR, VARSTR(file->reference));
if (file.reference != NULL)
kvPut(fileKv, MANIFEST_KEY_REFERENCE_VAR, VARSTR(file.reference));
if (file->sizeRepo != file->size)
kvPut(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, VARUINT64(file->sizeRepo));
if (file.sizeRepo != file.size)
kvPut(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, VARUINT64(file.sizeRepo));
kvPut(fileKv, MANIFEST_KEY_SIZE_VAR, VARUINT64(file->size));
kvPut(fileKv, MANIFEST_KEY_TIMESTAMP_VAR, VARUINT64((uint64_t)file->timestamp));
kvPut(fileKv, MANIFEST_KEY_SIZE_VAR, VARUINT64(file.size));
kvPut(fileKv, MANIFEST_KEY_TIMESTAMP_VAR, VARUINT64((uint64_t)file.timestamp));
if (!varEq(manifestOwnerVar(file->user), saveData->userDefault))
kvPut(fileKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(file->user));
if (!varEq(manifestOwnerVar(file.user), saveData->userDefault))
kvPut(fileKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(file.user));
infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, file->name, jsonFromKv(fileKv));
infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, file.name, jsonFromKv(fileKv));
MEM_CONTEXT_TEMP_RESET(1000);
}
@ -2437,22 +2593,22 @@ manifestValidate(Manifest *this, bool strict)
// Validate files
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
{
const ManifestFile *file = manifestFile(this, fileIdx);
const ManifestFile file = manifestFile(this, fileIdx);
// All files must have a checksum
if (file->checksumSha1[0] == '\0')
strCatFmt(error, "\nmissing checksum for file '%s'", strZ(file->name));
if (file.checksumSha1[0] == '\0')
strCatFmt(error, "\nmissing checksum for file '%s'", strZ(file.name));
// These are strict checks to be performed only after a backup and before the final manifest save
if (strict)
{
// Zero-length files must have a specific checksum
if (file->size == 0 && !strEqZ(HASH_TYPE_SHA1_ZERO_STR, file->checksumSha1))
strCatFmt(error, "\ninvalid checksum '%s' for zero size file '%s'", file->checksumSha1, strZ(file->name));
if (file.size == 0 && !strEqZ(HASH_TYPE_SHA1_ZERO_STR, file.checksumSha1))
strCatFmt(error, "\ninvalid checksum '%s' for zero size file '%s'", file.checksumSha1, strZ(file.name));
// Non-zero size files must have non-zero repo size
if (file->sizeRepo == 0 && file->size != 0)
strCatFmt(error, "\nrepo size must be > 0 for file '%s'", strZ(file->name));
if (file.sizeRepo == 0 && file.size != 0)
strCatFmt(error, "\nrepo size must be > 0 for file '%s'", strZ(file.name));
}
}
@ -2490,8 +2646,8 @@ manifestDbFind(const Manifest *this, const String *name)
/***********************************************************************************************************************************
File functions and getters/setters
***********************************************************************************************************************************/
const ManifestFile *
manifestFileFind(const Manifest *this, const String *name)
static ManifestFilePack **
manifestFilePackFindInternal(const Manifest *this, const String *name)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST, this);
@ -2501,12 +2657,26 @@ manifestFileFind(const Manifest *this, const String *name)
ASSERT(this != NULL);
ASSERT(name != NULL);
const ManifestFile *result = lstFind(this->pub.fileList, &name);
ManifestFilePack **const filePack = lstFind(this->pub.fileList, &name);
if (result == NULL)
if (filePack == NULL)
THROW_FMT(AssertError, "unable to find '%s' in manifest file list", strZ(name));
FUNCTION_TEST_RETURN(result);
FUNCTION_TEST_RETURN(filePack);
}
const ManifestFilePack *
manifestFilePackFind(const Manifest *this, const String *name)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST, this);
FUNCTION_TEST_PARAM(STRING, name);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(name != NULL);
FUNCTION_TEST_RETURN(*manifestFilePackFindInternal(this, name));
}
void
@ -2528,8 +2698,9 @@ manifestFileRemove(const Manifest *this, const String *name)
void
manifestFileUpdate(
Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference,
bool checksumPage, bool checksumPageError, const VariantList *checksumPageErrorList)
Manifest *const this, const String *const name, const uint64_t size, const uint64_t sizeRepo, const char *const checksumSha1,
const Variant *const reference, const bool checksumPage, const bool checksumPageError,
const String *const checksumPageErrorList)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(MANIFEST, this);
@ -2540,7 +2711,7 @@ manifestFileUpdate(
FUNCTION_TEST_PARAM(VARIANT, reference);
FUNCTION_TEST_PARAM(BOOL, checksumPage);
FUNCTION_TEST_PARAM(BOOL, checksumPageError);
FUNCTION_TEST_PARAM(VARIANT_LIST, checksumPageErrorList);
FUNCTION_TEST_PARAM(STRING, checksumPageErrorList);
FUNCTION_TEST_END();
ASSERT(this != NULL);
@ -2549,33 +2720,32 @@ manifestFileUpdate(
(!checksumPage && !checksumPageError && checksumPageErrorList == NULL) ||
(checksumPage && !checksumPageError && checksumPageErrorList == NULL) || (checksumPage && checksumPageError));
ManifestFile *file = (ManifestFile *)manifestFileFind(this, name);
ManifestFilePack **const filePack = manifestFilePackFindInternal(this, name);
ManifestFile file = manifestFileUnpack(*filePack);
MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
// Update reference if set
if (reference != NULL)
{
// Update reference if set
if (reference != NULL)
{
if (varStr(reference) == NULL)
file->reference = NULL;
else
file->reference = strLstAddIfMissing(this->referenceList, varStr(reference));
}
// Update checksum if set
if (checksumSha1 != NULL)
memcpy(file->checksumSha1, checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
// Update repo size
file->size = size;
file->sizeRepo = sizeRepo;
// Update checksum page info
file->checksumPage = checksumPage;
file->checksumPageError = checksumPageError;
file->checksumPageErrorList = varLstDup(checksumPageErrorList);
if (varStr(reference) == NULL)
file.reference = NULL;
else
file.reference = strLstAddIfMissing(this->referenceList, varStr(reference));
}
MEM_CONTEXT_END();
// Update checksum if set
if (checksumSha1 != NULL)
memcpy(file.checksumSha1, checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
// Update repo size
file.size = size;
file.sizeRepo = sizeRepo;
// Update checksum page info
file.checksumPage = checksumPage;
file.checksumPageError = checksumPageError;
file.checksumPageErrorList = checksumPageErrorList;
manifestFilePackUpdate(this, filePack, &file);
FUNCTION_TEST_RETURN_VOID();
}

53
src/info/manifest.h
View File

@ -96,7 +96,7 @@ typedef struct ManifestFile
bool checksumPageError:1; // Is there an error in the page checksum?
mode_t mode; // File mode
char checksumSha1[HASH_TYPE_SHA1_SIZE_HEX + 1]; // SHA1 checksum
const VariantList *checksumPageErrorList; // List of page checksum errors if there are any
const String *checksumPageErrorList; // List of page checksum errors if there are any
const String *user; // User name
const String *group; // Group name
const String *reference; // Reference to a prior backup
@ -258,21 +258,52 @@ manifestDbTotal(const Manifest *const this)
/***********************************************************************************************************************************
File functions and getters/setters
***********************************************************************************************************************************/
__attribute__((always_inline)) static inline const ManifestFile *
manifestFile(const Manifest *const this, const unsigned int fileIdx)
typedef struct ManifestFilePack ManifestFilePack;
// Unpack file pack returned by manifestFilePackGet()
ManifestFile manifestFileUnpack(const ManifestFilePack *filePack);
// Get file in pack format by index
__attribute__((always_inline)) static inline const ManifestFilePack *
manifestFilePackGet(const Manifest *const this, const unsigned int fileIdx)
{
return lstGet(THIS_PUB(Manifest)->fileList, fileIdx);
return *(ManifestFilePack **)lstGet(THIS_PUB(Manifest)->fileList, fileIdx);
}
void manifestFileAdd(Manifest *this, const ManifestFile *file);
const ManifestFile *manifestFileFind(const Manifest *this, const String *name);
// Get file name
__attribute__((always_inline)) static inline const String *
manifestFileNameGet(const Manifest *const this, const unsigned int fileIdx)
{
return (const String *)manifestFilePackGet(this, fileIdx);
}
// If the file requested is not found in the list, return the default passed rather than throw an error
__attribute__((always_inline)) static inline const ManifestFile *
manifestFileFindDefault(const Manifest *const this, const String *const name, const ManifestFile *const fileDefault)
// Get file by index
__attribute__((always_inline)) static inline ManifestFile
manifestFile(const Manifest *const this, const unsigned int fileIdx)
{
return manifestFileUnpack(manifestFilePackGet(this, fileIdx));
}
// Add a file
void manifestFileAdd(Manifest *this, ManifestFile file);
// Find file in pack format by name
const ManifestFilePack *manifestFilePackFind(const Manifest *this, const String *name);
// Find file by name
__attribute__((always_inline)) static inline ManifestFile
manifestFileFind(const Manifest *const this, const String *const name)
{
ASSERT_INLINE(name != NULL);
return lstFindDefault(THIS_PUB(Manifest)->fileList, &name, (void *)fileDefault);
return manifestFileUnpack(manifestFilePackFind(this, name));
}
// Does the file exist?
__attribute__((always_inline)) static inline bool
manifestFileExists(const Manifest *const this, const String *const name)
{
ASSERT_INLINE(name != NULL);
return lstFindDefault(THIS_PUB(Manifest)->fileList, &name, NULL) != NULL;
}
void manifestFileRemove(const Manifest *this, const String *name);
@ -286,7 +317,7 @@ manifestFileTotal(const Manifest *const this)
// Update a file with new data
void manifestFileUpdate(
Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference,
bool checksumPage, bool checksumPageError, const VariantList *checksumPageErrorList);
bool checksumPage, bool checksumPageError, const String *checksumPageErrorList);
/***********************************************************************************************************************************
Link functions and getters/setters

2
test/define.yaml
View File

@ -73,7 +73,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-convert
total: 11
total: 12
coverage:
- common/type/convert

48
test/src/module/command/backupTest.c
View File

@ -23,7 +23,7 @@ typedef struct TestBackupValidateCallbackData
{
const Storage *storage; // Storage object when needed (e.g. fileCompressed = true)
const String *path; // Subpath when storage is specified
const Manifest *manifest; // Manifest to check for files/links/paths
Manifest *manifest; // Manifest to check for files/links/paths
const ManifestData *manifestData; // Manifest data
String *content; // String where content should be added
} TestBackupValidateCallbackData;
@ -76,30 +76,31 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
// Check against the manifest
// ---------------------------------------------------------------------------------------------------------------------
const ManifestFile *file = manifestFileFind(data->manifest, manifestName);
ManifestFilePack **const filePack = manifestFilePackFindInternal(data->manifest, manifestName);
ManifestFile file = manifestFileUnpack(*filePack);
// Test size and repo-size. If compressed then set the repo-size to size so it will not be in test output. Even the same
// compression algorithm can give slightly different results based on the version so repo-size is not deterministic for
// compression.
if (size != file->size)
if (size != file.size)
THROW_FMT(AssertError, "'%s' size does match manifest", strZ(manifestName));
if (info->size != file->sizeRepo)
if (info->size != file.sizeRepo)
THROW_FMT(AssertError, "'%s' repo size does match manifest", strZ(manifestName));
if (data->manifestData->backupOptionCompressType != compressTypeNone)
((ManifestFile *)file)->sizeRepo = file->size;
file.sizeRepo = file.size;
// Test the checksum. pg_control and WAL headers have different checksums depending on cpu architecture so remove
// the checksum from the test output.
if (!strEqZ(checksum, file->checksumSha1))
if (!strEqZ(checksum, file.checksumSha1))
THROW_FMT(AssertError, "'%s' checksum does match manifest", strZ(manifestName));
if (strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL) ||
strBeginsWith(
manifestName, strNewFmt(MANIFEST_TARGET_PGDATA "/%s/", strZ(pgWalPath(data->manifestData->pgVersion)))))
{
((ManifestFile *)file)->checksumSha1[0] = '\0';
file.checksumSha1[0] = '\0';
}
// Test mode, user, group. These values are not in the manifest but we know what they should be based on the default
@ -113,6 +114,9 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
if (!strEq(info->group, TEST_GROUP_STR))
THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(manifestName));
// Update changes to manifest file
manifestFilePackUpdate(data->manifest, filePack, &file);
break;
}
@ -1310,7 +1314,7 @@ testRun(void)
manifestTargetAdd(manifestResume, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")});
manifestPathAdd(manifestResume, &(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR});
manifestFileAdd(manifestResume, &(ManifestFile){.name = STRDEF("pg_data/" PG_FILE_PGVERSION)});
manifestFileAdd(manifestResume, (ManifestFile){.name = STRDEF("pg_data/" PG_FILE_PGVERSION)});
manifestSave(
manifestResume,
@ -1430,7 +1434,7 @@ testRun(void)
OBJ_NEW_BEGIN(Manifest)
{
manifest = manifestNewInternal();
manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF("pg_data/test")});
manifestFileAdd(manifest, (ManifestFile){.name = STRDEF("pg_data/test")});
}
OBJ_NEW_END();
@ -1748,9 +1752,12 @@ testRun(void)
storagePg(), PG_FILE_PGVERSION, storageRepoWrite(),
strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION", strZ(resumeLabel))));
strcpy(
((ManifestFile *)manifestFileFind(manifestResume, STRDEF("pg_data/PG_VERSION")))->checksumSha1,
"06d06bb31b570b94d7b4325f511f853dbe771c21");
ManifestFilePack **const filePack = manifestFilePackFindInternal(manifestResume, STRDEF("pg_data/PG_VERSION"));
ManifestFile file = manifestFileUnpack(*filePack);
strcpy(file.checksumSha1, "06d06bb31b570b94d7b4325f511f853dbe771c21");
manifestFilePackUpdate(manifestResume, filePack, &file);
// Save the resume manifest
manifestSave(
@ -1841,14 +1848,19 @@ testRun(void)
HRN_STORAGE_PUT_EMPTY(
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/global/pg_control.gz", strZ(resumeLabel))));
((ManifestFile *)manifestFileFind(manifestResume, STRDEF("pg_data/global/pg_control")))->checksumSha1[0] = 0;
ManifestFilePack **const filePack = manifestFilePackFindInternal(manifestResume, STRDEF("pg_data/global/pg_control"));
ManifestFile file = manifestFileUnpack(*filePack);
file.checksumSha1[0] = 0;
manifestFilePackUpdate(manifestResume, filePack, &file);
// Size does not match between cluster and resume manifest
HRN_STORAGE_PUT_Z(storagePgWrite(), "size-mismatch", "TEST", .timeModified = backupTimeStart);
HRN_STORAGE_PUT_EMPTY(
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/size-mismatch.gz", strZ(resumeLabel))));
manifestFileAdd(
manifestResume, &(ManifestFile){
manifestResume, (ManifestFile){
.name = STRDEF("pg_data/size-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
.size = 33});
@ -1857,7 +1869,7 @@ testRun(void)
HRN_STORAGE_PUT_EMPTY(
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch.gz", strZ(resumeLabel))));
manifestFileAdd(
manifestResume, &(ManifestFile){
manifestResume, (ManifestFile){
.name = STRDEF("pg_data/time-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", .size = 4,
.timestamp = backupTimeStart - 1});
@ -1867,7 +1879,7 @@ testRun(void)
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/zero-size.gz", strZ(resumeLabel))),
"ZERO-SIZE");
manifestFileAdd(
manifestResume, &(ManifestFile){.name = STRDEF("pg_data/zero-size"), .size = 0, .timestamp = backupTimeStart});
manifestResume, (ManifestFile){.name = STRDEF("pg_data/zero-size"), .size = 0, .timestamp = backupTimeStart});
// Path is not in manifest
HRN_STORAGE_PATH_CREATE(
@ -2028,7 +2040,7 @@ testRun(void)
HRN_STORAGE_PUT_EMPTY(
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/resume-ref.gz", strZ(resumeLabel))));
manifestFileAdd(
manifestResume, &(ManifestFile){.name = STRDEF("pg_data/resume-ref"), .size = 0, .reference = STRDEF("BOGUS")});
manifestResume, (ManifestFile){.name = STRDEF("pg_data/resume-ref"), .size = 0, .reference = STRDEF("BOGUS")});
// Time does not match between cluster and resume manifest (but resume because time is in future so delta enabled). Note
// also that the repo file is intenionally corrupt to generate a warning about corruption in the repository.
@ -2036,7 +2048,7 @@ testRun(void)
HRN_STORAGE_PUT_EMPTY(
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch2.gz", strZ(resumeLabel))));
manifestFileAdd(
manifestResume, &(ManifestFile){
manifestResume, (ManifestFile){
.name = STRDEF("pg_data/time-mismatch2"), .checksumSha1 = "984816fd329622876e14907634264e6f332e9fb3", .size = 4,
.timestamp = backupTimeStart});

58
test/src/module/command/restoreTest.c
View File

@ -131,7 +131,7 @@ testManifestMinimal(const String *label, unsigned int pgVersion, const String *p
manifestPathAdd(result, &pathBase);
ManifestFile fileVersion = {
.name = STRDEF("pg_data/" PG_FILE_PGVERSION), .mode = 0600, .group = groupName(), .user = userName()};
manifestFileAdd(result, &fileVersion);
manifestFileAdd(result, fileVersion);
}
OBJ_NEW_END();
@ -815,7 +815,7 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/pg_hba.conf"), .size = 4, .timestamp = 1482182860});
(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/pg_hba.conf"), .size = 4, .timestamp = 1482182860});
TEST_RESULT_VOID(restoreManifestMap(manifest), "remap links");
@ -982,7 +982,7 @@ testRun(void)
ManifestPath path = {.name = STRDEF("pg_data/bogus_path"), .user = STRDEF("path-user-bogus")};
manifestPathAdd(manifest, &path);
ManifestFile file = {.name = STRDEF("pg_data/bogus_file"), .mode = 0600, .group = STRDEF("file-group-bogus")};
manifestFileAdd(manifest, &file);
manifestFileAdd(manifest, file);
ManifestLink link = {.name = STRDEF("pg_data/bogus_link"), .destination = STRDEF("/"), .group = STRDEF("link-group-bogus")};
manifestLinkAdd(manifest, &link);
@ -1034,7 +1034,7 @@ testRun(void)
manifest = testManifestMinimal(STRDEF("20161219-212741F_20161219-21275D"), PG_VERSION_96, pgPath);
manifestFileAdd(manifest, &file);
manifestFileAdd(manifest, file);
manifestLinkAdd(manifest, &link);
TEST_RESULT_VOID(restoreManifestOwner(manifest, &rootReplaceUser, &rootReplaceGroup), "check ownership");
@ -1051,7 +1051,7 @@ testRun(void)
TEST_TITLE("owner is root and ownership of pg_data is bad");
manifestPathAdd(manifest, &path);
manifestFileAdd(manifest, &file);
manifestFileAdd(manifest, file);
HRN_SYSTEM_FMT("sudo chown 77777:77777 %s", strZ(pgPath));
@ -1243,7 +1243,7 @@ testRun(void)
HRN_SYSTEM_FMT("rm -rf %s/*", strZ(pgPath));
manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)});
manifestFileAdd(manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)});
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_POSTGRESQLAUTOCONF);
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_RECOVERYSIGNAL);
@ -1303,7 +1303,7 @@ testRun(void)
manifest->pub.data.pgCatalogVersion = hrnPgCatalogVersion(PG_VERSION_90);
manifestTargetAdd(manifest, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")});
manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION)});
manifestFileAdd(manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION)});
}
OBJ_NEW_END();
@ -1324,11 +1324,11 @@ testRun(void)
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("user-made-system-db"), .id = 16380, .lastSystemId = 12168});
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF(UTF8_DB_NAME), .id = 16384, .lastSystemId = 12168});
manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/1/" PG_FILE_PGVERSION)});
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/1/" PG_FILE_PGVERSION)});
manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16381/" PG_FILE_PGVERSION)});
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16381/" PG_FILE_PGVERSION)});
manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16385/" PG_FILE_PGVERSION)});
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16385/" PG_FILE_PGVERSION)});
}
MEM_CONTEXT_END();
@ -1353,7 +1353,7 @@ testRun(void)
MEM_CONTEXT_BEGIN(manifest->pub.memContext)
{
manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16384/" PG_FILE_PGVERSION)});
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16384/" PG_FILE_PGVERSION)});
}
MEM_CONTEXT_END();
@ -1424,7 +1424,7 @@ testRun(void)
{
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test2"), .id = 32768, .lastSystemId = 12168});
manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)});
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)});
}
MEM_CONTEXT_END();
@ -1448,7 +1448,7 @@ testRun(void)
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387"), .tablespaceId = 16387, .tablespaceName = STRDEF("ts1"),
.path = STRDEF("/ts1")});
manifestFileAdd(
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)});
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)});
}
MEM_CONTEXT_END();
@ -1470,7 +1470,7 @@ testRun(void)
{
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test3"), .id = 65536, .lastSystemId = 12168});
manifestFileAdd(
manifest, &(ManifestFile){
manifest, (ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387/PG_9.4_201409291/65536/" PG_FILE_PGVERSION)});
}
MEM_CONTEXT_END();
@ -2053,7 +2053,7 @@ testRun(void)
// PG_VERSION
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"});
@ -2184,7 +2184,7 @@ testRun(void)
// tablespace_map (will be ignored during restore)
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP);
@ -2218,7 +2218,7 @@ testRun(void)
// pg_tblspc/1/16384/PG_VERSION
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/1/16384/" PG_FILE_PGVERSION), .size = 4,
.timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"});
@ -2400,7 +2400,7 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL), .size = 8192, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "5e2b96c19c4f5c63a5afa2de504d29fe64a4c908"});
@ -2409,7 +2409,7 @@ testRun(void)
// global/999
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/999"), .size = 0, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = HASH_TYPE_SHA1_ZERO, .reference = STRDEF(TEST_LABEL)});
@ -2418,7 +2418,7 @@ testRun(void)
// PG_VERSION
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
@ -2439,7 +2439,7 @@ testRun(void)
// base/1/PG_VERSION
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
@ -2452,7 +2452,7 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/2"), .size = 8192, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "4d7b2a36c5387decf799352a3751883b7ceb96aa"});
@ -2470,7 +2470,7 @@ testRun(void)
// base/16384/PG_VERSION
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/16384/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
@ -2483,7 +2483,7 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/16384/16385"), .size = 16384, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "d74e5f7ebe52a3ed468ba08c5b6aefaccd1ca88f"});
@ -2499,7 +2499,7 @@ testRun(void)
// base/32768/PG_VERSION
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/32768/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
@ -2512,7 +2512,7 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/32768/32769"), .size = 32768, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "a40f0986acb1531ce0cc75a23dcf8aa406ae9081"});
@ -2529,7 +2529,7 @@ testRun(void)
.name = name, .destination = STRDEF("../config/postgresql.conf"), .group = groupName(), .user = userName()});
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA "postgresql.conf"), .size = 15, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "98b8abb2e681e2a5a7d8ab082c0a79727887558d"});
@ -2546,7 +2546,7 @@ testRun(void)
.name = name, .destination = STRDEF("../config/pg_hba.conf"), .group = groupName(), .user = userName()});
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA "pg_hba.conf"), .size = 11, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "401215e092779574988a854d8c7caed7f91dba4b"});
@ -2555,7 +2555,7 @@ testRun(void)
// tablespace_map (will be ignored during restore)
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP);

31
test/src/module/common/typeConvertTest.c
View File

@ -202,5 +202,36 @@ testRun(void)
TEST_RESULT_UINT(cvtZToUInt64("18446744073709551615"), 18446744073709551615U, "convert string to uint64");
}
// *****************************************************************************************************************************
if (testBegin("cvtUInt64ToVarInt128() and cvtUInt64FromVarInt128()"))
{
uint8_t buffer[CVT_VARINT128_BUFFER_SIZE];
size_t bufferPos;
bufferPos = 0;
TEST_ERROR(cvtUInt64ToVarInt128(9999, buffer, &bufferPos, 1), AssertError, "buffer overflow");
bufferPos = 0;
TEST_RESULT_VOID(cvtUInt64ToVarInt128(0xFFFFFFFFFFFFFFFF, buffer, &bufferPos, sizeof(buffer)), "to varint-128");
TEST_RESULT_UINT(bufferPos, 10, "check buffer position");
bufferPos = 0;
TEST_RESULT_UINT(cvtUInt64FromVarInt128(buffer, &bufferPos), 0xFFFFFFFFFFFFFFFF, "to uint64");
TEST_RESULT_UINT(bufferPos, 10, "check buffer position");
bufferPos = 0;
TEST_RESULT_VOID(cvtUInt64ToVarInt128(0, buffer, &bufferPos, sizeof(buffer)), "to varint-128");
TEST_RESULT_UINT(bufferPos, 1, "check buffer position");
bufferPos = 0;
TEST_RESULT_UINT(cvtUInt64FromVarInt128(buffer, &bufferPos), 0, "to uint64");
TEST_RESULT_UINT(bufferPos, 1, "check buffer position");
uint8_t buffer2[CVT_VARINT128_BUFFER_SIZE + 1];
memset(buffer2, 0xFF, sizeof(buffer2));
bufferPos = 0;
TEST_ERROR(cvtUInt64FromVarInt128(buffer2, &bufferPos), FormatError, "unterminated varint-128 integer");
}
FUNCTION_HARNESS_RETURN_VOID();
}

50
test/src/module/info/manifestTest.c
View File

@ -950,7 +950,7 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860});
(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860});
TEST_RESULT_VOID(manifestBuildValidate(manifest, false, 1482182860, false), "validate manifest");
TEST_RESULT_INT(manifest->pub.data.backupTimestampCopyStart, 1482182860, "check copy start");
@ -1029,22 +1029,22 @@ testRun(void)
&(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR, .mode = 0700, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/BOGUS"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
}
@ -1059,17 +1059,17 @@ testRun(void)
manifestFileAdd(
manifestPrior,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
.checksumSha1 = "da39a3ee5e6b4b0d3255bfef95601890afd80709"});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182860,
.checksumSha1 = "ccccccccccaaaaaaaaaabbbbbbbbbbdddddddddd"});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
}
@ -1109,18 +1109,18 @@ testRun(void)
lstClear(manifest->pub.fileList);
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
@ -1160,7 +1160,7 @@ testRun(void)
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182859,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
@ -1171,11 +1171,11 @@ testRun(void)
varLstAdd(checksumPageErrorList, varNewUInt(77));
manifestFileAdd(
manifestPrior,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", .checksumPage = true, .checksumPageError = true,
.checksumPageErrorList = checksumPageErrorList});
.checksumPageErrorList = jsonFromVar(varNewVarLst(checksumPageErrorList))});
TEST_RESULT_VOID(manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, NULL), "incremental manifest");
@ -1211,18 +1211,18 @@ testRun(void)
lstClear(manifest->pub.fileList);
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifestFileAdd(
manifestPrior,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.reference = STRDEF("20190101-010101F_20190202-010101D"),
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
@ -1279,14 +1279,14 @@ testRun(void)
lstClear(manifest->pub.fileList);
manifestFileAdd(
manifest,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
manifest->pub.data.backupOptionOnline = BOOL_TRUE_VAR;
manifestFileAdd(
manifestPrior,
&(ManifestFile){
(ManifestFile){
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
@ -1729,18 +1729,14 @@ testRun(void)
TEST_TITLE("manifest getters");
// ManifestFile getters
const ManifestFile *file = NULL;
ManifestFile file = {0};
TEST_ERROR(manifestFileFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest file list");
TEST_ASSIGN(file, manifestFileFind(manifest, STRDEF("pg_data/PG_VERSION")), "manifestFileFind()");
TEST_RESULT_STR_Z(file->name, "pg_data/PG_VERSION", "find file");
TEST_RESULT_STR_Z(file.name, "pg_data/PG_VERSION", "find file");
TEST_RESULT_STR_Z(
manifestFileFindDefault(manifest, STRDEF("bogus"), file)->name, "pg_data/PG_VERSION",
"manifestFileFindDefault() - return default");
TEST_RESULT_STR_Z(
manifestFileFind(manifest, STRDEF("pg_data/special-@#!$^&*()_+~`{}[]\\:;"))->name,
manifestFileFind(manifest, STRDEF("pg_data/special-@#!$^&*()_+~`{}[]\\:;")).name,
"pg_data/special-@#!$^&*()_+~`{}[]\\:;", "find special file");
TEST_ASSIGN(file, manifestFileFindDefault(manifest, STRDEF("bogus"), NULL), "manifestFileFindDefault()");
TEST_RESULT_PTR(file, NULL, "return default NULL");
TEST_RESULT_BOOL(manifestFileExists(manifest, STRDEF("bogus")), false, "manifest file does not exist");
TEST_RESULT_VOID(
manifestFileUpdate(manifest, STRDEF("pg_data/postgresql.conf"), 4457, 4457, "", NULL, false, false, NULL),

4
test/src/module/performance/typeTest.c
View File

@ -317,8 +317,8 @@ testRun(void)
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
const ManifestFile *file = manifestFile(manifest, fileIdx);
ASSERT(file == manifestFileFind(manifest, file->name));
const ManifestFile file = manifestFile(manifest, fileIdx);
ASSERT(strEq(file.name, manifestFileFind(manifest, file.name).name));
}
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));