diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 0088b8a5e..9f15463f5 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -68,6 +68,7 @@ + diff --git a/src/command/backup/backup.c b/src/command/backup/backup.c index 35a541d80..bf62371aa 100644 --- a/src/command/backup/backup.c +++ b/src/command/backup/backup.c @@ -576,37 +576,43 @@ void backupResumeCallback(void *data, const StorageInfo *info) if (fileCompressType != compressTypeNone) manifestName = compressExtStrip(manifestName, fileCompressType); - // Find the file in both manifests - const ManifestFile *file = manifestFileFindDefault(resumeData->manifest, manifestName, NULL); - const ManifestFile *fileResume = manifestFileFindDefault(resumeData->manifestResume, manifestName, NULL); - // Check if the file can be resumed or must be removed const char *removeReason = NULL; if (fileCompressType != resumeData->compressType) removeReason = "mismatched compression type"; - else if (file == NULL) + else if (!manifestFileExists(resumeData->manifest, manifestName)) removeReason = "missing in manifest"; - else if (file->reference != NULL) - removeReason = "reference in manifest"; - else if (fileResume == NULL) - removeReason = "missing in resumed manifest"; - else if (fileResume->reference != NULL) - removeReason = "reference in resumed manifest"; - else if (fileResume->checksumSha1[0] == '\0') - removeReason = "no checksum in resumed manifest"; - else if (file->size != fileResume->size) - removeReason = "mismatched size"; - else if (!resumeData->delta && file->timestamp != fileResume->timestamp) - removeReason = "mismatched timestamp"; - else if (file->size == 0) - // ??? don't resume zero size files because Perl wouldn't -- this can be removed after the migration) - removeReason = "zero size"; else { - manifestFileUpdate( - resumeData->manifest, manifestName, file->size, fileResume->sizeRepo, fileResume->checksumSha1, NULL, - fileResume->checksumPage, fileResume->checksumPageError, fileResume->checksumPageErrorList); + const ManifestFile file = manifestFileFind(resumeData->manifest, manifestName); + + if (file.reference != NULL) + removeReason = "reference in manifest"; + else if (!manifestFileExists(resumeData->manifestResume, manifestName)) + removeReason = "missing in resumed manifest"; + else + { + const ManifestFile fileResume = manifestFileFind(resumeData->manifestResume, manifestName); + + if (fileResume.reference != NULL) + removeReason = "reference in resumed manifest"; + else if (fileResume.checksumSha1[0] == '\0') + removeReason = "no checksum in resumed manifest"; + else if (file.size != fileResume.size) + removeReason = "mismatched size"; + else if (!resumeData->delta && file.timestamp != fileResume.timestamp) + removeReason = "mismatched timestamp"; + else if (file.size == 0) + // ??? don't resume zero size files because Perl wouldn't -- this can be removed after the migration) + removeReason = "zero size"; + else + { + manifestFileUpdate( + resumeData->manifest, manifestName, file.size, fileResume.sizeRepo, fileResume.checksumSha1, NULL, + fileResume.checksumPage, fileResume.checksumPageError, fileResume.checksumPageErrorList); + } + } } // Remove the file if it could not be resumed @@ -981,7 +987,7 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti file.checksumSha1, strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))), HASH_TYPE_SHA1_SIZE_HEX + 1); - manifestFileAdd(manifest, &file); + manifestFileAdd(manifest, file); LOG_DETAIL_FMT("wrote '%s' file returned from pg_stop_backup()", strZ(name)); } @@ -1165,7 +1171,7 @@ backupJobResult( { MEM_CONTEXT_TEMP_BEGIN() { - const ManifestFile *const file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job))); + const ManifestFile file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job))); const unsigned int processId = protocolParallelJobProcessId(job); PackRead *const jobResult = protocolParallelJobResult(job); @@ -1205,7 +1211,7 @@ backupJobResult( else if (copyResult == backupCopyResultSkip) { LOG_DETAIL_PID_FMT(processId, "skip file removed by database %s", strZ(fileLog)); - strLstAdd(fileRemove, file->name); + strLstAdd(fileRemove, file.name); } // Else file was copied so update manifest else @@ -1220,13 +1226,13 @@ backupJobResult( " continue but this may be an issue unless the resumed backup path in the repository is known to be" " corrupted.\n" "NOTE: this does not indicate a problem with the PostgreSQL page checksums.", - strZ(file->name), file->checksumSha1); + strZ(file.name), file.checksumSha1); } LOG_DETAIL_PID_FMT(processId, "backup file %s (%s)%s", strZ(fileLog), strZ(logProgress), strZ(logChecksum)); // If the file had page checksums calculated during the copy - ASSERT((!file->checksumPage && checksumPageResult == NULL) || (file->checksumPage && checksumPageResult != NULL)); + ASSERT((!file.checksumPage && checksumPageResult == NULL) || (file.checksumPage && checksumPageResult != NULL)); bool checksumPageError = false; const VariantList *checksumPageErrorList = NULL; @@ -1300,8 +1306,8 @@ backupJobResult( // Update file info and remove any reference to the file's existence in a prior backup manifestFileUpdate( - manifest, file->name, copySize, repoSize, strZ(copyChecksum), VARSTR(NULL), file->checksumPage, - checksumPageError, checksumPageErrorList); + manifest, file.name, copySize, repoSize, strZ(copyChecksum), VARSTR(NULL), file.checksumPage, + checksumPageError, checksumPageErrorList != NULL ? jsonFromVar(varNewVarLst(checksumPageErrorList)) : NULL); } } MEM_CONTEXT_TEMP_END(); @@ -1420,14 +1426,18 @@ backupProcessQueueComparator(const void *item1, const void *item2) ASSERT(item1 != NULL); ASSERT(item2 != NULL); + // Unpack files + ManifestFile file1 = manifestFileUnpack(*(const ManifestFilePack **)item1); + ManifestFile file2 = manifestFileUnpack(*(const ManifestFilePack **)item2); + // If the size differs then that's enough to determine order - if ((*(ManifestFile **)item1)->size < (*(ManifestFile **)item2)->size) + if (file1.size < file2.size) FUNCTION_TEST_RETURN(-1); - else if ((*(ManifestFile **)item1)->size > (*(ManifestFile **)item2)->size) + else if (file1.size > file2.size) FUNCTION_TEST_RETURN(1); // If size is the same then use name to generate a deterministic ordering (names must be unique) - FUNCTION_TEST_RETURN(strCmp((*(ManifestFile **)item1)->name, (*(ManifestFile **)item2)->name)); + FUNCTION_TEST_RETURN(strCmp(file1.name, file2.name)); } // Helper to generate the backup queues @@ -1479,20 +1489,21 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) { - const ManifestFile *file = manifestFile(manifest, fileIdx); + const ManifestFilePack *const filePack = manifestFilePackGet(manifest, fileIdx); + const ManifestFile file = manifestFileUnpack(filePack); // If the file is a reference it should only be backed up if delta and not zero size - if (file->reference != NULL && (!jobData->delta || file->size == 0)) + if (file.reference != NULL && (!jobData->delta || file.size == 0)) continue; // Is pg_control in the backup? - if (strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))) + if (strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))) pgControlFound = true; // Files that must be copied from the primary are always put in queue 0 when backup from standby - if (jobData->backupStandby && backupProcessFilePrimary(jobData->standbyExp, file->name)) + if (jobData->backupStandby && backupProcessFilePrimary(jobData->standbyExp, file.name)) { - lstAdd(*(List **)lstGet(jobData->queueList, 0), &file); + lstAdd(*(List **)lstGet(jobData->queueList, 0), &filePack); } // Else find the correct queue by matching the file to a target else @@ -1504,7 +1515,7 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData) { CHECK(AssertError, targetIdx < strLstSize(targetList), "backup target not found"); - if (strBeginsWith(file->name, strLstGet(targetList, targetIdx))) + if (strBeginsWith(file.name, strLstGet(targetList, targetIdx))) break; targetIdx++; @@ -1512,11 +1523,11 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData) while (1); // Add file to queue - lstAdd(*(List **)lstGet(jobData->queueList, targetIdx + queueOffset), &file); + lstAdd(*(List **)lstGet(jobData->queueList, targetIdx + queueOffset), &filePack); } // Add size to total - result += file->size; + result += file.size; // Increment total files fileTotal++; @@ -1600,21 +1611,21 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx if (!lstEmpty(queue)) { - const ManifestFile *file = *(ManifestFile **)lstGet(queue, 0); + const ManifestFile file = manifestFileUnpack(*(ManifestFilePack **)lstGet(queue, 0)); // Create backup job ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE); PackWrite *const param = protocolCommandParam(command); - pckWriteStrP(param, manifestPathPg(file->name)); - pckWriteBoolP(param, !strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))); - pckWriteU64P(param, file->size); - pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file->name)); - pckWriteStrP(param, file->checksumSha1[0] != 0 ? STR(file->checksumSha1) : NULL); - pckWriteBoolP(param, file->checksumPage); + pckWriteStrP(param, manifestPathPg(file.name)); + pckWriteBoolP(param, !strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))); + pckWriteU64P(param, file.size); + pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file.name)); + pckWriteStrP(param, file.checksumSha1[0] != 0 ? STR(file.checksumSha1) : NULL); + pckWriteBoolP(param, file.checksumPage); pckWriteU64P(param, jobData->lsnStart); - pckWriteStrP(param, file->name); - pckWriteBoolP(param, file->reference != NULL); + pckWriteStrP(param, file.name); + pckWriteBoolP(param, file.reference != NULL); pckWriteU32P(param, jobData->compressType); pckWriteI32P(param, jobData->compressLevel); pckWriteStrP(param, jobData->backupLabel); @@ -1628,7 +1639,7 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx // Assign job to result MEM_CONTEXT_PRIOR_BEGIN() { - result = protocolParallelJobNew(VARSTR(file->name), command); + result = protocolParallelJobNew(VARSTR(file.name), command); } MEM_CONTEXT_PRIOR_END(); @@ -1774,7 +1785,7 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart backupStandby && protocolParallelJobProcessId(job) > 1 ? backupData->hostStandby : backupData->hostPrimary, storagePathP( protocolParallelJobProcessId(job) > 1 ? storagePgIdx(pgIdx) : backupData->storagePrimary, - manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job)))->name)), fileRemove, job, + manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job))).name)), fileRemove, job, sizeTotal, &sizeProgress); } @@ -1814,22 +1825,22 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) { - const ManifestFile *const file = manifestFile(manifest, fileIdx); + const ManifestFile file = manifestFile(manifest, fileIdx); // If the file has a reference, then it was not copied since it can be retrieved from the referenced backup. However, // if hardlinking is enabled the link will need to be created. - if (file->reference != NULL) + if (file.reference != NULL) { // If hardlinking is enabled then create a hardlink for files that have not changed since the last backup if (hardLink) { - LOG_DETAIL_FMT("hardlink %s to %s", strZ(file->name), strZ(file->reference)); + LOG_DETAIL_FMT("hardlink %s to %s", strZ(file.name), strZ(file.reference)); const String *const linkName = storagePathP( - storageRepo(), strNewFmt("%s/%s%s", strZ(backupPathExp), strZ(file->name), compressExt)); - const String *const linkDestination = storagePathP( + storageRepo(), strNewFmt("%s/%s%s", strZ(backupPathExp), strZ(file.name), compressExt)); + const String *const linkDestination = storagePathP( storageRepo(), - strNewFmt(STORAGE_REPO_BACKUP "/%s/%s%s", strZ(file->reference), strZ(file->name), compressExt)); + strNewFmt(STORAGE_REPO_BACKUP "/%s/%s%s", strZ(file.reference), strZ(file.name), compressExt)); THROW_ON_SYS_ERROR_FMT( link(strZ(linkDestination), strZ(linkName)) == -1, FileOpenError, @@ -1838,7 +1849,7 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart // Else log the reference. With delta, it is possible that references may have been removed if a file needed to be // recopied. else - LOG_DETAIL_FMT("reference %s to %s", strZ(file->name), strZ(file->reference)); + LOG_DETAIL_FMT("reference %s to %s", strZ(file.name), strZ(file.reference)); } } @@ -1974,7 +1985,7 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif memcpy(file.checksumSha1, strZ(strSubN(archiveFile, 25, 40)), HASH_TYPE_SHA1_SIZE_HEX + 1); - manifestFileAdd(manifest, &file); + manifestFileAdd(manifest, file); } } MEM_CONTEXT_TEMP_END(); diff --git a/src/command/info/info.c b/src/command/info/info.c index 8b481d78d..7a177fcd6 100644 --- a/src/command/info/info.c +++ b/src/command/info/info.c @@ -533,10 +533,10 @@ backupListAdd( for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(repoData->manifest); fileIdx++) { - const ManifestFile *file = manifestFile(repoData->manifest, fileIdx); + const ManifestFile file = manifestFile(repoData->manifest, fileIdx); - if (file->checksumPageError) - varLstAdd(checksumPageErrorList, varNewStr(manifestPathPg(file->name))); + if (file.checksumPageError) + varLstAdd(checksumPageErrorList, varNewStr(manifestPathPg(file.name))); } if (!varLstEmpty(checksumPageErrorList)) diff --git a/src/command/restore/restore.c b/src/command/restore/restore.c index 89c862cee..bbac67663 100644 --- a/src/command/restore/restore.c +++ b/src/command/restore/restore.c @@ -533,22 +533,22 @@ restoreManifestMap(Manifest *manifest) if (target.name == NULL) { // Is the specified link a file or a path? Error if they both match. - const ManifestPath *const path = manifestPathFindDefault(manifest, manifestName, NULL); - const ManifestFile *const file = manifestFileFindDefault(manifest, manifestName, NULL); + const bool pathExists = manifestPathFindDefault(manifest, manifestName, NULL) != NULL; + const bool fileExists = manifestFileExists(manifest, manifestName); - CHECK(FormatError, path == NULL || file == NULL, "link may not be both file and path"); + CHECK(FormatError, !pathExists || !fileExists, "link may not be both file and path"); target = (ManifestTarget){.name = manifestName, .path = linkPath, .type = manifestTargetTypeLink}; // If a file - if (file != NULL) + if (fileExists) { // File needs to be set so the file/path is updated later but set it to something invalid just in case it // it does not get updated due to a regression target.file = DOT_STR; } // Else error if not a path - else if (path == NULL) + else if (!pathExists) { THROW_FMT( LinkMapError, @@ -666,20 +666,20 @@ restoreManifestOwnerReplace(const String *const owner, const String *const owner } // Helper to get list of owners from a file/link/path list -#define RESTORE_MANIFEST_OWNER_GET(type) \ +#define RESTORE_MANIFEST_OWNER_GET(type, deref) \ for (unsigned int itemIdx = 0; itemIdx < manifest##type##Total(manifest); itemIdx++) \ { \ - Manifest##type *item = (Manifest##type *)manifest##type(manifest, itemIdx); \ + Manifest##type item = deref manifest##type(manifest, itemIdx); \ \ - if (item->user == NULL) \ + if (item.user == NULL) \ userNull = true; \ else \ - strLstAddIfMissing(userList, item->user); \ + strLstAddIfMissing(userList, item.user); \ \ - if (item->group == NULL) \ + if (item.group == NULL) \ groupNull = true; \ else \ - strLstAddIfMissing(groupList, item->group); \ + strLstAddIfMissing(groupList, item.group); \ } // Helper to warn when an owner is missing and must be remapped @@ -719,9 +719,9 @@ restoreManifestOwner(const Manifest *const manifest, const String **const rootRe bool groupNull = false; StringList *groupList = strLstNew(); - RESTORE_MANIFEST_OWNER_GET(File); - RESTORE_MANIFEST_OWNER_GET(Link); - RESTORE_MANIFEST_OWNER_GET(Path); + RESTORE_MANIFEST_OWNER_GET(File, ); + RESTORE_MANIFEST_OWNER_GET(Link, *(ManifestLink *)); + RESTORE_MANIFEST_OWNER_GET(Path, *(ManifestPath *)); // Update users and groups in the manifest (this can only be done as root) // ------------------------------------------------------------------------------------------------------------------------- @@ -918,14 +918,15 @@ restoreCleanInfoListCallback(void *data, const StorageInfo *info) { case storageTypeFile: { - const ManifestFile *manifestFile = manifestFileFindDefault(cleanData->manifest, manifestName, NULL); - - if (manifestFile != NULL && manifestLinkFindDefault(cleanData->manifest, manifestName, NULL) == NULL) + if (manifestFileExists(cleanData->manifest, manifestName) && + manifestLinkFindDefault(cleanData->manifest, manifestName, NULL) == NULL) { + const ManifestFile manifestFile = manifestFileFind(cleanData->manifest, manifestName); + restoreCleanOwnership( - pgPath, manifestFile->user, cleanData->rootReplaceUser, manifestFile->group, cleanData->rootReplaceGroup, + pgPath, manifestFile.user, cleanData->rootReplaceUser, manifestFile.group, cleanData->rootReplaceGroup, info->userId, info->groupId, false); - restoreCleanMode(pgPath, manifestFile->mode, info); + restoreCleanMode(pgPath, manifestFile.mode, info); } else { @@ -1133,7 +1134,7 @@ restoreCleanBuild(const Manifest *const manifest, const String *const rootReplac // Skip the tablespace_map file when present so PostgreSQL does not rewrite links in pg_tblspc. The tablespace links will be // created after paths are cleaned. - if (manifestFileFindDefault(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_TABLESPACEMAP), NULL) != NULL && + if (manifestFileExists(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_TABLESPACEMAP)) && manifestData(manifest)->pgVersion >= PG_VERSION_TABLESPACE_MAP) { LOG_DETAIL_FMT("skip '" PG_FILE_TABLESPACEMAP "' -- tablespace links will be created based on mappings"); @@ -1142,7 +1143,7 @@ restoreCleanBuild(const Manifest *const manifest, const String *const rootReplac // Skip postgresql.auto.conf if preserve is set and the PostgreSQL version supports recovery GUCs if (manifestData(manifest)->pgVersion >= PG_VERSION_RECOVERY_GUC && cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_PRESERVE && - manifestFileFindDefault(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF), NULL) != NULL) + manifestFileExists(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF))) { LOG_DETAIL_FMT("skip '" PG_FILE_POSTGRESQLAUTOCONF "' -- recovery type is preserve"); manifestFileRemove(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)); @@ -1341,11 +1342,11 @@ restoreSelectiveExpression(Manifest *manifest) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) { - const ManifestFile *file = manifestFile(manifest, fileIdx); + const String *const fileName = manifestFileNameGet(manifest, fileIdx); - if (regExpMatch(baseRegExp, file->name) || regExpMatch(tablespaceRegExp, file->name)) + if (regExpMatch(baseRegExp, fileName) || regExpMatch(tablespaceRegExp, fileName)) { - String *dbId = strBase(strPath(file->name)); + String *dbId = strBase(strPath(fileName)); // In the highly unlikely event that a system database was somehow added after the backup began, it will only be // found in the file list and not the manifest db section, so add it to the system database list @@ -1907,14 +1908,18 @@ restoreProcessQueueComparator(const void *item1, const void *item2) ASSERT(item1 != NULL); ASSERT(item2 != NULL); + // Unpack files + ManifestFile file1 = manifestFileUnpack(*(const ManifestFilePack **)item1); + ManifestFile file2 = manifestFileUnpack(*(const ManifestFilePack **)item2); + // If the size differs then that's enough to determine order - if ((*(ManifestFile **)item1)->size < (*(ManifestFile **)item2)->size) + if (file1.size < file2.size) FUNCTION_TEST_RETURN(-1); - else if ((*(ManifestFile **)item1)->size > (*(ManifestFile **)item2)->size) + else if (file1.size > file2.size) FUNCTION_TEST_RETURN(1); // If size is the same then use name to generate a deterministic ordering (names must be unique) - FUNCTION_TEST_RETURN(strCmp((*(ManifestFile **)item1)->name, (*(ManifestFile **)item2)->name)); + FUNCTION_TEST_RETURN(strCmp(file1.name, file2.name)); } static uint64_t @@ -1960,7 +1965,8 @@ restoreProcessQueue(Manifest *manifest, List **queueList) // Now put all files into the processing queues for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) { - const ManifestFile *file = manifestFile(manifest, fileIdx); + const ManifestFilePack *const filePack = manifestFilePackGet(manifest, fileIdx); + const ManifestFile file = manifestFileUnpack(filePack); // Find the target that contains this file unsigned int targetIdx = 0; @@ -1970,7 +1976,7 @@ restoreProcessQueue(Manifest *manifest, List **queueList) // A target should always be found CHECK(FormatError, targetIdx < strLstSize(targetList), "backup target not found"); - if (strBeginsWith(file->name, strLstGet(targetList, targetIdx))) + if (strBeginsWith(file.name, strLstGet(targetList, targetIdx))) break; targetIdx++; @@ -1978,10 +1984,10 @@ restoreProcessQueue(Manifest *manifest, List **queueList) while (1); // Add file to queue - lstAdd(*(List **)lstGet(*queueList, targetIdx), &file); + lstAdd(*(List **)lstGet(*queueList, targetIdx), &filePack); // Add size to total - result += file->size; + result += file.size; } // Sort the queues @@ -2052,8 +2058,8 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer { MEM_CONTEXT_TEMP_BEGIN() { - const ManifestFile *file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job))); - bool zeroed = restoreFileZeroed(file->name, zeroExp); + const ManifestFile file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job))); + bool zeroed = restoreFileZeroed(file.name, zeroExp); bool copy = pckReadBoolP(protocolParallelJobResult(job)); String *log = strCatZ(strNew(), "restore"); @@ -2063,7 +2069,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer strCatZ(log, " zeroed"); // Add filename - strCatFmt(log, " file %s", strZ(restoreFilePgPath(manifest, file->name))); + strCatFmt(log, " file %s", strZ(restoreFilePgPath(manifest, file.name))); // If not copied and not zeroed add details to explain why it was not copied if (!copy && !zeroed) @@ -2074,8 +2080,8 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer if (cfgOptionBool(cfgOptForce)) { strCatFmt( - log, "exists and matches size %" PRIu64 " and modification time %" PRIu64, file->size, - (uint64_t)file->timestamp); + log, "exists and matches size %" PRIu64 " and modification time %" PRIu64, file.size, + (uint64_t)file.timestamp); } // Else a checksum delta or file is zero-length else @@ -2083,7 +2089,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer strCatZ(log, "exists and "); // No need to copy zero-length files - if (file->size == 0) + if (file.size == 0) { strCatZ(log, "is zero size"); } @@ -2094,12 +2100,12 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer } // Add size and percent complete - sizeRestored += file->size; - strCatFmt(log, " (%s, %" PRIu64 "%%)", strZ(strSizeFormat(file->size)), sizeRestored * 100 / sizeTotal); + sizeRestored += file.size; + strCatFmt(log, " (%s, %" PRIu64 "%%)", strZ(strSizeFormat(file.size)), sizeRestored * 100 / sizeTotal); // If not zero-length add the checksum - if (file->size != 0 && !zeroed) - strCatFmt(log, " checksum %s", file->checksumSha1); + if (file.size != 0 && !zeroed) + strCatFmt(log, " checksum %s", file.checksumSha1); LOG_DETAIL_PID(protocolParallelJobProcessId(job), strZ(log)); } @@ -2178,24 +2184,24 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId if (!lstEmpty(queue)) { - const ManifestFile *file = *(ManifestFile **)lstGet(queue, 0); + const ManifestFile file = manifestFileUnpack(*(ManifestFilePack **)lstGet(queue, 0)); // Create restore job ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE); PackWrite *const param = protocolCommandParam(command); - pckWriteStrP(param, file->name); + pckWriteStrP(param, file.name); pckWriteU32P(param, jobData->repoIdx); - pckWriteStrP(param, file->reference != NULL ? file->reference : manifestData(jobData->manifest)->backupLabel); + pckWriteStrP(param, file.reference != NULL ? file.reference : manifestData(jobData->manifest)->backupLabel); pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType); - pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file->name)); - pckWriteStrP(param, STR(file->checksumSha1)); - pckWriteBoolP(param, restoreFileZeroed(file->name, jobData->zeroExp)); - pckWriteU64P(param, file->size); - pckWriteTimeP(param, file->timestamp); - pckWriteModeP(param, file->mode); - pckWriteStrP(param, restoreManifestOwnerReplace(file->user, jobData->rootReplaceUser)); - pckWriteStrP(param, restoreManifestOwnerReplace(file->group, jobData->rootReplaceGroup)); + pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file.name)); + pckWriteStrP(param, STR(file.checksumSha1)); + pckWriteBoolP(param, restoreFileZeroed(file.name, jobData->zeroExp)); + pckWriteU64P(param, file.size); + pckWriteTimeP(param, file.timestamp); + pckWriteModeP(param, file.mode); + pckWriteStrP(param, restoreManifestOwnerReplace(file.user, jobData->rootReplaceUser)); + pckWriteStrP(param, restoreManifestOwnerReplace(file.group, jobData->rootReplaceGroup)); pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart); pckWriteBoolP(param, cfgOptionBool(cfgOptDelta)); pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce)); @@ -2207,7 +2213,7 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId // Assign job to result MEM_CONTEXT_PRIOR_BEGIN() { - result = protocolParallelJobNew(VARSTR(file->name), command); + result = protocolParallelJobNew(VARSTR(file.name), command); } MEM_CONTEXT_PRIOR_END(); diff --git a/src/command/verify/verify.c b/src/command/verify/verify.c index 74e4ab7e0..bd1b0f431 100644 --- a/src/command/verify/verify.c +++ b/src/command/verify/verify.c @@ -902,7 +902,7 @@ verifyBackup(void *data) { do { - const ManifestFile *fileData = manifestFile(jobData->manifest, jobData->manifestFileIdx); + const ManifestFile fileData = manifestFile(jobData->manifest, jobData->manifestFileIdx); String *filePathName = NULL; @@ -910,16 +910,16 @@ verifyBackup(void *data) backupResult->totalFileVerify++; // Check if the file is referenced in a prior backup - if (fileData->reference != NULL) + if (fileData.reference != NULL) { // If the prior backup is not in the result list, then that backup was never processed (likely due to the --set // option) so verify the file - unsigned int backupPriorIdx = lstFindIdx(jobData->backupResultList, &fileData->reference); + unsigned int backupPriorIdx = lstFindIdx(jobData->backupResultList, &fileData.reference); if (backupPriorIdx == LIST_NOT_FOUND) { filePathName = strNewFmt( - STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name), + STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData.reference), strZ(fileData.name), strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); } // Else the backup this file references has a result so check the processing state for the referenced backup @@ -931,14 +931,14 @@ verifyBackup(void *data) if (!backupResultPrior->fileVerifyComplete) { filePathName = strNewFmt( - STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name), + STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData.reference), strZ(fileData.name), strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); } // Else skip verification else { String *priorFile = strNewFmt( - "%s/%s%s", strZ(fileData->reference), strZ(fileData->name), + "%s/%s%s", strZ(fileData.reference), strZ(fileData.name), strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); unsigned int backupPriorInvalidIdx = lstFindIdx(backupResultPrior->invalidFileList, &priorFile); @@ -965,7 +965,7 @@ verifyBackup(void *data) else { filePathName = strNewFmt( - STORAGE_REPO_BACKUP "/%s/%s%s", strZ(backupResult->backupLabel), strZ(fileData->name), + STORAGE_REPO_BACKUP "/%s/%s%s", strZ(backupResult->backupLabel), strZ(fileData.name), strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); } @@ -978,8 +978,8 @@ verifyBackup(void *data) pckWriteStrP(param, filePathName); // If the checksum is not present in the manifest, it will be calculated by manifest load - pckWriteStrP(param, STR(fileData->checksumSha1)); - pckWriteU64P(param, fileData->size); + pckWriteStrP(param, STR(fileData.checksumSha1)); + pckWriteU64P(param, fileData.size); pckWriteStrP(param, jobData->backupCipherPass); // Assign job to result (prepend backup label being processed to the key since some files are in a prior backup) diff --git a/src/common/type/convert.c b/src/common/type/convert.c index 13af3ca1a..eaa849c99 100644 --- a/src/common/type/convert.c +++ b/src/common/type/convert.c @@ -482,3 +482,80 @@ cvtZToUInt64(const char *value) FUNCTION_TEST_RETURN(cvtZToUInt64Base(value, 10)); } + +/**********************************************************************************************************************************/ +void +cvtUInt64ToVarInt128(uint64_t value, uint8_t *const buffer, size_t *const bufferPos, const size_t bufferSize) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT64, value); + FUNCTION_TEST_PARAM_P(VOID, buffer); + FUNCTION_TEST_PARAM(UINT64, bufferSize); + FUNCTION_TEST_END(); + + ASSERT(buffer != NULL); + ASSERT(bufferPos != NULL); + ASSERT(bufferSize > *bufferPos); + + // Keep encoding bytes while the remaining value is greater than 7 bits + while (value >= 0x80) + { + // Encode the lower order 7 bits, adding the continuation bit to indicate there is more data + buffer[*bufferPos] = (unsigned char)value | 0x80; + + // Shift the value to remove bits that have been encoded + value >>= 7; + + // Keep track of size so we know how many bytes to write out + (*bufferPos)++; + + // Make sure the buffer won't overflow + if (*bufferPos >= bufferSize) + THROW(AssertError, "buffer overflow"); + } + + // Encode the last 7 bits of value + buffer[*bufferPos] = (unsigned char)value; + (*bufferPos)++; + + FUNCTION_TEST_RETURN_VOID(); +} + +uint64_t +cvtUInt64FromVarInt128(const uint8_t *const value, size_t *const valuePos) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM_P(VOID, value); + FUNCTION_TEST_PARAM_P(UINT64, valuePos); + FUNCTION_TEST_END(); + + ASSERT(value != NULL); + ASSERT(valuePos != NULL); + + // Decode all bytes + uint64_t result = 0; + uint8_t byte; + + for (unsigned int valueIdx = 0; valueIdx < CVT_VARINT128_BUFFER_SIZE; valueIdx++) + { + // Get the next encoded byte + byte = value[*valuePos]; + + // Shift the lower order 7 encoded bits into the uint64 in reverse order + result |= (uint64_t)(byte & 0x7f) << (7 * valueIdx); + + // Increment value position to indicate that the byte has been processed + (*valuePos)++; + + // Done if the high order bit is not set to indicate more data + if (byte < 0x80) + break; + } + + // By this point all bytes should have been read so error if this is not the case. This could be due to a coding error or + // corrupton in the data stream. + if (byte >= 0x80) + THROW(FormatError, "unterminated varint-128 integer"); + + FUNCTION_TEST_RETURN(result); +} diff --git a/src/common/type/convert.h b/src/common/type/convert.h index bd1f4afc8..e1f1ecfe0 100644 --- a/src/common/type/convert.h +++ b/src/common/type/convert.h @@ -15,6 +15,7 @@ Required buffer sizes ***********************************************************************************************************************************/ #define CVT_BOOL_BUFFER_SIZE 6 #define CVT_BASE10_BUFFER_SIZE 64 +#define CVT_VARINT128_BUFFER_SIZE 10 /*********************************************************************************************************************************** Functions @@ -85,6 +86,10 @@ size_t cvtUInt64ToZ(uint64_t value, char *buffer, size_t bufferSize); uint64_t cvtZToUInt64(const char *value); uint64_t cvtZToUInt64Base(const char *value, int base); +// Convert uint64 to base-128 varint and vice versa +void cvtUInt64ToVarInt128(uint64_t value, uint8_t *buffer, size_t *bufferPos, size_t bufferSize); +uint64_t cvtUInt64FromVarInt128(const uint8_t *value, size_t *valuePos); + // Convert boolean to zero-terminated string. Use cvtBoolToConstZ() whenever possible since it is more efficient. size_t cvtBoolToZ(bool value, char *buffer, size_t bufferSize); const char *cvtBoolToConstZ(bool value); diff --git a/src/common/type/pack.c b/src/common/type/pack.c index bf3984608..a93881e9c 100644 --- a/src/common/type/pack.c +++ b/src/common/type/pack.c @@ -104,11 +104,6 @@ Array and object types: #include "common/type/convert.h" #include "common/type/pack.h" -/*********************************************************************************************************************************** -Constants -***********************************************************************************************************************************/ -#define PACK_UINT64_SIZE_MAX 10 - /*********************************************************************************************************************************** Map PackType types to the types that will be written into the pack. This hides the details of the type IDs from the user and allows the IDs used in the pack to differ from the IDs the user sees. @@ -495,7 +490,7 @@ pckReadU64Internal(PackRead *this) uint8_t byte; // Convert bytes from varint-128 encoding to a uint64 - for (unsigned int bufferIdx = 0; bufferIdx < PACK_UINT64_SIZE_MAX; bufferIdx++) + for (unsigned int bufferIdx = 0; bufferIdx < CVT_VARINT128_BUFFER_SIZE; bufferIdx++) { // Get the next encoded byte pckReadBuffer(this, 1); @@ -1409,7 +1404,7 @@ pckWriteBuffer(PackWrite *this, const Buffer *buffer) Pack an unsigned 64-bit integer to base-128 varint encoding ***********************************************************************************************************************************/ static void -pckWriteU64Internal(PackWrite *this, uint64_t value) +pckWriteU64Internal(PackWrite *const this, const uint64_t value) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(PACK_WRITE, this); @@ -1418,27 +1413,13 @@ pckWriteU64Internal(PackWrite *this, uint64_t value) ASSERT(this != NULL); - unsigned char buffer[PACK_UINT64_SIZE_MAX]; + unsigned char buffer[CVT_VARINT128_BUFFER_SIZE]; size_t size = 0; - // Convert uint64 to varint-128 encoding. Keep writing out bytes while the remaining value is greater than 7 bits. - while (value >= 0x80) - { - // Encode the lower order 7 bits, adding the continuation bit to indicate there is more data - buffer[size] = (unsigned char)value | 0x80; - - // Shift the value to remove bits that have been encoded - value >>= 7; - - // Keep track of size so we know how many bytes to write out - size++; - } - - // Encode the last 7 bits of value - buffer[size] = (unsigned char)value; + cvtUInt64ToVarInt128(value, buffer, &size, sizeof(buffer)); // Write encoded bytes to the buffer - pckWriteBuffer(this, BUF(buffer, size + 1)); + pckWriteBuffer(this, BUF(buffer, size)); FUNCTION_TEST_RETURN_VOID(); } diff --git a/src/info/infoBackup.c b/src/info/infoBackup.c index 26bdfa953..225cb695b 100644 --- a/src/info/infoBackup.c +++ b/src/info/infoBackup.c @@ -370,22 +370,22 @@ infoBackupDataAdd(const InfoBackup *this, const Manifest *manifest) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) { - const ManifestFile *file = manifestFile(manifest, fileIdx); + const ManifestFile file = manifestFile(manifest, fileIdx); - backupSize += file->size; - backupRepoSize += file->sizeRepo > 0 ? file->sizeRepo : file->size; + backupSize += file.size; + backupRepoSize += file.sizeRepo > 0 ? file.sizeRepo : file.size; // If a reference to a file exists, then it is in a previous backup and the delta calculation was already done - if (file->reference != NULL) - strLstAddIfMissing(referenceList, file->reference); + if (file.reference != NULL) + strLstAddIfMissing(referenceList, file.reference); else { - backupSizeDelta += file->size; - backupRepoSizeDelta += file->sizeRepo > 0 ? file->sizeRepo : file->size; + backupSizeDelta += file.size; + backupRepoSizeDelta += file.sizeRepo > 0 ? file.sizeRepo : file.size; } // Is there an error in the file? - if (file->checksumPageError) + if (file.checksumPageError) backupError = true; } diff --git a/src/info/manifest.c b/src/info/manifest.c index 90c8c8a4a..81322f538 100644 --- a/src/info/manifest.c +++ b/src/info/manifest.c @@ -198,59 +198,208 @@ manifestDbAdd(Manifest *this, const ManifestDb *db) FUNCTION_TEST_RETURN_VOID(); } +// Base time used as a delta to reduce the size of packed timestamps. This will be set on the first call to manifestFilePack(). +static time_t manifestPackBaseTime = -1; + +// Flags used to reduce the size of packed data. They should be ordered from most to least likely and can be reordered at will. +typedef enum +{ + manifestFilePackFlagReference, + manifestFilePackFlagChecksumPage, + manifestFilePackFlagChecksumPageError, + manifestFilePackFlagChecksumPageErrorList, +} ManifestFilePackFlag; + +// Pack file into a compact format to save memory +static ManifestFilePack * +manifestFilePack(const ManifestFile *const file) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST_FILE, file); + FUNCTION_TEST_END(); + + uint8_t buffer[512]; + size_t bufferPos = 0; + + // Flags + uint64_t flag = 0; + + if (file->checksumPage) + flag |= 1 << manifestFilePackFlagChecksumPage; + + if (file->checksumPageError) + flag |= 1 << manifestFilePackFlagChecksumPageError; + + if (file->checksumPageErrorList != NULL) + flag |= 1 << manifestFilePackFlagChecksumPageErrorList; + + if (file->reference != NULL) + flag |= 1 << manifestFilePackFlagReference; + + cvtUInt64ToVarInt128(flag, buffer, &bufferPos, sizeof(buffer)); + + // Size + cvtUInt64ToVarInt128(file->size, buffer, &bufferPos, sizeof(buffer)); + + // Use the first timestamp that appears as the base for all other timestamps. Ideally we would like a timestamp as close to the + // middle as possible but it doesn't seem worth doing the calculation. + if (manifestPackBaseTime == -1) + manifestPackBaseTime = file->timestamp; + + // Timestamp + cvtUInt64ToVarInt128(cvtInt64ToZigZag(manifestPackBaseTime - file->timestamp), buffer, &bufferPos, sizeof(buffer)); + + // SHA1 checksum + strcpy((char *)buffer + bufferPos, file->checksumSha1); + bufferPos += HASH_TYPE_SHA1_SIZE_HEX + 1; + + // Reference + if (file->reference != NULL) + cvtUInt64ToVarInt128((uintptr_t)file->reference, buffer, &bufferPos, sizeof(buffer)); + + // Mode + cvtUInt64ToVarInt128(file->mode, buffer, &bufferPos, sizeof(buffer)); + + // User/group + cvtUInt64ToVarInt128((uintptr_t)file->user, buffer, &bufferPos, sizeof(buffer)); + cvtUInt64ToVarInt128((uintptr_t)file->group, buffer, &bufferPos, sizeof(buffer)); + + // Repo size + cvtUInt64ToVarInt128(file->sizeRepo, buffer, &bufferPos, sizeof(buffer)); + + // Allocate memory for the file pack + uint8_t *const result = memNew( + sizeof(StringPub) + strSize(file->name) + 1 + bufferPos + (file->checksumPageErrorList != NULL ? + sizeof(StringPub) + strSize(file->checksumPageErrorList) + 1 : 0)); + + // Create string object for the file name + *(StringPub *)result = (StringPub){.size = (unsigned int)strSize(file->name), .buffer = (char *)result + sizeof(StringPub)}; + size_t resultPos = sizeof(StringPub); + + memcpy(result + resultPos, (uint8_t *)strZ(file->name), strSize(file->name) + 1); + resultPos += strSize(file->name) + 1; + + // Copy pack data + memcpy(result + resultPos, buffer, bufferPos); + + // Create string object for the checksum error list + if (file->checksumPageErrorList != NULL) + { + resultPos += bufferPos; + + *(StringPub *)(result + resultPos) = (StringPub) + {.size = (unsigned int)strSize(file->checksumPageErrorList), .buffer = (char *)result + resultPos + sizeof(StringPub)}; + resultPos += sizeof(StringPub); + + memcpy(result + resultPos, (uint8_t *)strZ(file->checksumPageErrorList), strSize(file->checksumPageErrorList) + 1); + } + + FUNCTION_TEST_RETURN((ManifestFilePack *)result); +} + +ManifestFile +manifestFileUnpack(const ManifestFilePack *const filePack) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM_P(VOID, filePack); + FUNCTION_TEST_END(); + + ASSERT(filePack != NULL); + ASSERT(manifestPackBaseTime != -1); + + ManifestFile result = {0}; + size_t bufferPos = 0; + + // Name + result.name = (const String *)filePack; + bufferPos += sizeof(StringPub) + strSize(result.name) + 1; + + // Flags + const uint64_t flag = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos); + + // Size + result.size = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos); + + // Timestamp + result.timestamp = + manifestPackBaseTime - (time_t)cvtInt64FromZigZag(cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos)); + + // Checksum page + result.checksumPage = flag & (1 << manifestFilePackFlagChecksumPage) ? true : false; + + // SHA1 checksum + memcpy(result.checksumSha1, (const uint8_t *)filePack + bufferPos, HASH_TYPE_SHA1_SIZE_HEX + 1); + bufferPos += HASH_TYPE_SHA1_SIZE_HEX + 1; + + // Reference + if (flag & (1 << manifestFilePackFlagReference)) + result.reference = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos); + + // Mode + result.mode = (mode_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos); + + // User/group + result.user = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos); + result.group = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos); + + // Repo size + result.sizeRepo = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos); + + // Checksum page error + result.checksumPageError = flag & (1 << manifestFilePackFlagChecksumPageError) ? true : false; + + if (flag & (1 << manifestFilePackFlagChecksumPageErrorList)) + result.checksumPageErrorList = (const String *)((const uint8_t *)filePack + bufferPos); + + FUNCTION_TEST_RETURN(result); +} + void -manifestFileAdd(Manifest *this, const ManifestFile *file) +manifestFileAdd(Manifest *this, ManifestFile file) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(VOID, file); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(file.name != NULL); + + file.user = manifestOwnerCache(this, file.user); + file.group = manifestOwnerCache(this, file.group); + + if (file.reference != NULL) + file.reference = strLstAddIfMissing(this->referenceList, file.reference); + + MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList)) + { + const ManifestFilePack *const filePack = manifestFilePack(&file); + lstAdd(this->pub.fileList, &filePack); + } + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +// Update file pack by creating a new one and then freeing the old one +static void +manifestFilePackUpdate(Manifest *const this, ManifestFilePack **const filePack, const ManifestFile *const file) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM_P(VOID, filePack); FUNCTION_TEST_PARAM(MANIFEST_FILE, file); FUNCTION_TEST_END(); ASSERT(this != NULL); + ASSERT(filePack != NULL); ASSERT(file != NULL); - ASSERT(file->name != NULL); MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList)) { - ManifestFile fileAdd = - { - .checksumPage = file->checksumPage, - .checksumPageError = file->checksumPageError, - .checksumPageErrorList = varLstDup(file->checksumPageErrorList), - .group = manifestOwnerCache(this, file->group), - .mode = file->mode, - .name = strDup(file->name), - .size = file->size, - .sizeRepo = file->sizeRepo, - .timestamp = file->timestamp, - .user = manifestOwnerCache(this, file->user), - }; - - memcpy(fileAdd.checksumSha1, file->checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1); - - if (file->reference != NULL) - { - // Search for the reference in the list - for (unsigned int referenceIdx = 0; referenceIdx < strLstSize(this->referenceList); referenceIdx++) - { - const String *found = strLstGet(this->referenceList, referenceIdx); - - if (strEq(file->reference, found)) - { - fileAdd.reference = found; - break; - } - } - - // If not found then add it - if (fileAdd.reference == NULL) - { - strLstAdd(this->referenceList, file->reference); - fileAdd.reference = strLstGet(this->referenceList, strLstSize(this->referenceList) - 1); - } - } - - lstAdd(this->pub.fileList, &fileAdd); + ManifestFilePack *const filePackOld = *filePack; + *filePack = manifestFilePack(file); + memFree(filePackOld); } MEM_CONTEXT_END(); @@ -364,7 +513,7 @@ manifestNewInternal(void) { .memContext = memContextCurrent(), .dbList = lstNewP(sizeof(ManifestDb), .comparator = lstComparatorStr), - .fileList = lstNewP(sizeof(ManifestFile), .comparator = lstComparatorStr), + .fileList = lstNewP(sizeof(ManifestFilePack *), .comparator = lstComparatorStr), .linkList = lstNewP(sizeof(ManifestLink), .comparator = lstComparatorStr), .pathList = lstNewP(sizeof(ManifestPath), .comparator = lstComparatorStr), .targetList = lstNewP(sizeof(ManifestTarget), .comparator = lstComparatorStr), @@ -843,7 +992,7 @@ manifestBuildCallback(void *data, const StorageInfo *info) !strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL); } - manifestFileAdd(buildData.manifest, &file); + manifestFileAdd(buildData.manifest, file); break; } @@ -1163,12 +1312,12 @@ manifestNewBuild( while (fileIdx < manifestFileTotal(this)) { // If this file looks like a relation. Note that this never matches on _init forks. - const ManifestFile *file = manifestFile(this, fileIdx); + const String *const filePathName = manifestFileNameGet(this, fileIdx); - if (regExpMatch(relationExp, file->name)) + if (regExpMatch(relationExp, filePathName)) { // Get the filename (without path) - const char *fileName = strBaseZ(file->name); + const char *fileName = strBaseZ(filePathName); size_t fileNameSize = strlen(fileName); // Strip off the numeric part of the relation @@ -1193,8 +1342,8 @@ manifestNewBuild( { // Determine if the relation is unlogged String *relationInit = strNewFmt( - "%.*s%s_init", (int)(strSize(file->name) - fileNameSize), strZ(file->name), relationFileId); - lastRelationFileIdUnlogged = manifestFileFindDefault(this, relationInit, NULL) != NULL; + "%.*s%s_init", (int)(strSize(filePathName) - fileNameSize), strZ(filePathName), relationFileId); + lastRelationFileIdUnlogged = manifestFileExists(this, relationInit); strFree(relationInit); // Save the file id so we don't need to do the lookup next time if it doesn't change @@ -1204,7 +1353,7 @@ manifestNewBuild( // If relation is unlogged then remove it if (lastRelationFileIdUnlogged) { - manifestFileRemove(this, file->name); + manifestFileRemove(this, filePathName); continue; } } @@ -1262,13 +1411,13 @@ manifestBuildValidate(Manifest *this, bool delta, time_t copyStart, CompressType { for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++) { - const ManifestFile *file = manifestFile(this, fileIdx); + const ManifestFile file = manifestFile(this, fileIdx); // Check for timestamp in the future - if (file->timestamp > copyStart) + if (file.timestamp > copyStart) { LOG_WARN_FMT( - "file '%s' has timestamp in the future, enabling delta checksum", strZ(manifestPathPg(file->name))); + "file '%s' has timestamp in the future, enabling delta checksum", strZ(manifestPathPg(file.name))); this->pub.data.backupOptionDelta = BOOL_TRUE_VAR; break; @@ -1337,29 +1486,30 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type { for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++) { - const ManifestFile *file = manifestFile(this, fileIdx); - const ManifestFile *filePrior = manifestFileFindDefault(manifestPrior, file->name, NULL); + const ManifestFile file = manifestFile(this, fileIdx); // If file was found in prior manifest then perform checks - if (filePrior != NULL) + if (manifestFileExists(manifestPrior, file.name)) { + const ManifestFile filePrior = manifestFileFind(manifestPrior, file.name); + // Check for timestamp earlier than the prior backup - if (file->timestamp < filePrior->timestamp) + if (file.timestamp < filePrior.timestamp) { LOG_WARN_FMT( "file '%s' has timestamp earlier than prior backup, enabling delta checksum", - strZ(manifestPathPg(file->name))); + strZ(manifestPathPg(file.name))); this->pub.data.backupOptionDelta = BOOL_TRUE_VAR; break; } // Check for size change with no timestamp change - if (file->size != filePrior->size && file->timestamp == filePrior->timestamp) + if (file.size != filePrior.size && file.timestamp == filePrior.timestamp) { LOG_WARN_FMT( "file '%s' has same timestamp as prior but different size, enabling delta checksum", - strZ(manifestPathPg(file->name))); + strZ(manifestPathPg(file.name))); this->pub.data.backupOptionDelta = BOOL_TRUE_VAR; break; @@ -1375,17 +1525,20 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type for (unsigned int fileIdx = 0; fileIdx < lstSize(this->pub.fileList); fileIdx++) { - const ManifestFile *file = manifestFile(this, fileIdx); - const ManifestFile *filePrior = manifestFileFindDefault(manifestPrior, file->name, NULL); + const ManifestFile file = manifestFile(this, fileIdx); // Check if prior file can be used - if (filePrior != NULL && file->size == filePrior->size && - (delta || file->size == 0 || file->timestamp == filePrior->timestamp)) + if (manifestFileExists(manifestPrior, file.name)) { - manifestFileUpdate( - this, file->name, file->size, filePrior->sizeRepo, filePrior->checksumSha1, - VARSTR(filePrior->reference != NULL ? filePrior->reference : manifestPrior->pub.data.backupLabel), - filePrior->checksumPage, filePrior->checksumPageError, filePrior->checksumPageErrorList); + const ManifestFile filePrior = manifestFileFind(manifestPrior, file.name); + + if (file.size == filePrior.size && (delta || file.size == 0 || file.timestamp == filePrior.timestamp)) + { + manifestFileUpdate( + this, file.name, file.size, filePrior.sizeRepo, filePrior.checksumSha1, + VARSTR(filePrior.reference != NULL ? filePrior.reference : manifestPrior->pub.data.backupLabel), + filePrior.checksumPage, filePrior.checksumPageError, filePrior.checksumPageErrorList); + } } } } @@ -1618,7 +1771,7 @@ manifestLoadCallback(void *callbackData, const String *section, const String *ke const Variant *checksumPageErrorList = kvGetDefault(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, NULL); if (checksumPageErrorList != NULL) - file.checksumPageErrorList = varVarLst(checksumPageErrorList); + file.checksumPageErrorList = jsonFromVar(checksumPageErrorList); } if (kvKeyExists(fileKv, MANIFEST_KEY_GROUP_VAR)) @@ -1640,7 +1793,7 @@ manifestLoadCallback(void *callbackData, const String *section, const String *ke } lstAdd(loadData->fileFoundList, &valueFound); - manifestFileAdd(manifest, &file); + manifestFileAdd(manifest, file); } MEM_CONTEXT_END(); } @@ -1926,17 +2079,20 @@ manifestNewLoad(IoRead *read) // Process file defaults for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++) { - ManifestFile *file = lstGet(this->pub.fileList, fileIdx); + ManifestFilePack **const filePack = lstGet(this->pub.fileList, fileIdx); + ManifestFile file = manifestFileUnpack(*filePack); ManifestLoadFound *found = lstGet(loadData.fileFoundList, fileIdx); if (!found->group) - file->group = manifestOwnerCache(this, manifestOwnerGet(loadData.fileGroupDefault)); + file.group = manifestOwnerCache(this, manifestOwnerGet(loadData.fileGroupDefault)); if (!found->mode) - file->mode = loadData.fileModeDefault; + file.mode = loadData.fileModeDefault; if (!found->user) - file->user = manifestOwnerCache(this, manifestOwnerGet(loadData.fileUserDefault)); + file.user = manifestOwnerCache(this, manifestOwnerGet(loadData.fileUserDefault)); + + manifestFilePackUpdate(this, filePack, &file); } // Process link defaults @@ -2243,41 +2399,41 @@ manifestSaveCallback(void *callbackData, const String *sectionNext, InfoSave *in { for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) { - const ManifestFile *file = manifestFile(manifest, fileIdx); + const ManifestFile file = manifestFile(manifest, fileIdx); KeyValue *fileKv = kvNew(); // Save if the file size is not zero and the checksum exists. The checksum might not exist if this is a partial // save performed during a backup. - if (file->size != 0 && file->checksumSha1[0] != 0) - kvPut(fileKv, MANIFEST_KEY_CHECKSUM_VAR, VARSTRZ(file->checksumSha1)); + if (file.size != 0 && file.checksumSha1[0] != 0) + kvPut(fileKv, MANIFEST_KEY_CHECKSUM_VAR, VARSTRZ(file.checksumSha1)); - if (file->checksumPage) + if (file.checksumPage) { - kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, VARBOOL(!file->checksumPageError)); + kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, VARBOOL(!file.checksumPageError)); - if (file->checksumPageErrorList != NULL) - kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, varNewVarLst(file->checksumPageErrorList)); + if (file.checksumPageErrorList != NULL) + kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, jsonToVar(file.checksumPageErrorList)); } - if (!varEq(manifestOwnerVar(file->group), saveData->groupDefault)) - kvPut(fileKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(file->group)); + if (!varEq(manifestOwnerVar(file.group), saveData->groupDefault)) + kvPut(fileKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(file.group)); - if (file->mode != saveData->fileModeDefault) - kvPut(fileKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", file->mode))); + if (file.mode != saveData->fileModeDefault) + kvPut(fileKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", file.mode))); - if (file->reference != NULL) - kvPut(fileKv, MANIFEST_KEY_REFERENCE_VAR, VARSTR(file->reference)); + if (file.reference != NULL) + kvPut(fileKv, MANIFEST_KEY_REFERENCE_VAR, VARSTR(file.reference)); - if (file->sizeRepo != file->size) - kvPut(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, VARUINT64(file->sizeRepo)); + if (file.sizeRepo != file.size) + kvPut(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, VARUINT64(file.sizeRepo)); - kvPut(fileKv, MANIFEST_KEY_SIZE_VAR, VARUINT64(file->size)); - kvPut(fileKv, MANIFEST_KEY_TIMESTAMP_VAR, VARUINT64((uint64_t)file->timestamp)); + kvPut(fileKv, MANIFEST_KEY_SIZE_VAR, VARUINT64(file.size)); + kvPut(fileKv, MANIFEST_KEY_TIMESTAMP_VAR, VARUINT64((uint64_t)file.timestamp)); - if (!varEq(manifestOwnerVar(file->user), saveData->userDefault)) - kvPut(fileKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(file->user)); + if (!varEq(manifestOwnerVar(file.user), saveData->userDefault)) + kvPut(fileKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(file.user)); - infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, file->name, jsonFromKv(fileKv)); + infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, file.name, jsonFromKv(fileKv)); MEM_CONTEXT_TEMP_RESET(1000); } @@ -2437,22 +2593,22 @@ manifestValidate(Manifest *this, bool strict) // Validate files for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++) { - const ManifestFile *file = manifestFile(this, fileIdx); + const ManifestFile file = manifestFile(this, fileIdx); // All files must have a checksum - if (file->checksumSha1[0] == '\0') - strCatFmt(error, "\nmissing checksum for file '%s'", strZ(file->name)); + if (file.checksumSha1[0] == '\0') + strCatFmt(error, "\nmissing checksum for file '%s'", strZ(file.name)); // These are strict checks to be performed only after a backup and before the final manifest save if (strict) { // Zero-length files must have a specific checksum - if (file->size == 0 && !strEqZ(HASH_TYPE_SHA1_ZERO_STR, file->checksumSha1)) - strCatFmt(error, "\ninvalid checksum '%s' for zero size file '%s'", file->checksumSha1, strZ(file->name)); + if (file.size == 0 && !strEqZ(HASH_TYPE_SHA1_ZERO_STR, file.checksumSha1)) + strCatFmt(error, "\ninvalid checksum '%s' for zero size file '%s'", file.checksumSha1, strZ(file.name)); // Non-zero size files must have non-zero repo size - if (file->sizeRepo == 0 && file->size != 0) - strCatFmt(error, "\nrepo size must be > 0 for file '%s'", strZ(file->name)); + if (file.sizeRepo == 0 && file.size != 0) + strCatFmt(error, "\nrepo size must be > 0 for file '%s'", strZ(file.name)); } } @@ -2490,8 +2646,8 @@ manifestDbFind(const Manifest *this, const String *name) /*********************************************************************************************************************************** File functions and getters/setters ***********************************************************************************************************************************/ -const ManifestFile * -manifestFileFind(const Manifest *this, const String *name) +static ManifestFilePack ** +manifestFilePackFindInternal(const Manifest *this, const String *name) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(MANIFEST, this); @@ -2501,12 +2657,26 @@ manifestFileFind(const Manifest *this, const String *name) ASSERT(this != NULL); ASSERT(name != NULL); - const ManifestFile *result = lstFind(this->pub.fileList, &name); + ManifestFilePack **const filePack = lstFind(this->pub.fileList, &name); - if (result == NULL) + if (filePack == NULL) THROW_FMT(AssertError, "unable to find '%s' in manifest file list", strZ(name)); - FUNCTION_TEST_RETURN(result); + FUNCTION_TEST_RETURN(filePack); +} + +const ManifestFilePack * +manifestFilePackFind(const Manifest *this, const String *name) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + FUNCTION_TEST_RETURN(*manifestFilePackFindInternal(this, name)); } void @@ -2528,8 +2698,9 @@ manifestFileRemove(const Manifest *this, const String *name) void manifestFileUpdate( - Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference, - bool checksumPage, bool checksumPageError, const VariantList *checksumPageErrorList) + Manifest *const this, const String *const name, const uint64_t size, const uint64_t sizeRepo, const char *const checksumSha1, + const Variant *const reference, const bool checksumPage, const bool checksumPageError, + const String *const checksumPageErrorList) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(MANIFEST, this); @@ -2540,7 +2711,7 @@ manifestFileUpdate( FUNCTION_TEST_PARAM(VARIANT, reference); FUNCTION_TEST_PARAM(BOOL, checksumPage); FUNCTION_TEST_PARAM(BOOL, checksumPageError); - FUNCTION_TEST_PARAM(VARIANT_LIST, checksumPageErrorList); + FUNCTION_TEST_PARAM(STRING, checksumPageErrorList); FUNCTION_TEST_END(); ASSERT(this != NULL); @@ -2549,33 +2720,32 @@ manifestFileUpdate( (!checksumPage && !checksumPageError && checksumPageErrorList == NULL) || (checksumPage && !checksumPageError && checksumPageErrorList == NULL) || (checksumPage && checksumPageError)); - ManifestFile *file = (ManifestFile *)manifestFileFind(this, name); + ManifestFilePack **const filePack = manifestFilePackFindInternal(this, name); + ManifestFile file = manifestFileUnpack(*filePack); - MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList)) + // Update reference if set + if (reference != NULL) { - // Update reference if set - if (reference != NULL) - { - if (varStr(reference) == NULL) - file->reference = NULL; - else - file->reference = strLstAddIfMissing(this->referenceList, varStr(reference)); - } - - // Update checksum if set - if (checksumSha1 != NULL) - memcpy(file->checksumSha1, checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1); - - // Update repo size - file->size = size; - file->sizeRepo = sizeRepo; - - // Update checksum page info - file->checksumPage = checksumPage; - file->checksumPageError = checksumPageError; - file->checksumPageErrorList = varLstDup(checksumPageErrorList); + if (varStr(reference) == NULL) + file.reference = NULL; + else + file.reference = strLstAddIfMissing(this->referenceList, varStr(reference)); } - MEM_CONTEXT_END(); + + // Update checksum if set + if (checksumSha1 != NULL) + memcpy(file.checksumSha1, checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1); + + // Update repo size + file.size = size; + file.sizeRepo = sizeRepo; + + // Update checksum page info + file.checksumPage = checksumPage; + file.checksumPageError = checksumPageError; + file.checksumPageErrorList = checksumPageErrorList; + + manifestFilePackUpdate(this, filePack, &file); FUNCTION_TEST_RETURN_VOID(); } diff --git a/src/info/manifest.h b/src/info/manifest.h index 6417725c9..d885f9114 100644 --- a/src/info/manifest.h +++ b/src/info/manifest.h @@ -96,7 +96,7 @@ typedef struct ManifestFile bool checksumPageError:1; // Is there an error in the page checksum? mode_t mode; // File mode char checksumSha1[HASH_TYPE_SHA1_SIZE_HEX + 1]; // SHA1 checksum - const VariantList *checksumPageErrorList; // List of page checksum errors if there are any + const String *checksumPageErrorList; // List of page checksum errors if there are any const String *user; // User name const String *group; // Group name const String *reference; // Reference to a prior backup @@ -258,21 +258,52 @@ manifestDbTotal(const Manifest *const this) /*********************************************************************************************************************************** File functions and getters/setters ***********************************************************************************************************************************/ -__attribute__((always_inline)) static inline const ManifestFile * -manifestFile(const Manifest *const this, const unsigned int fileIdx) +typedef struct ManifestFilePack ManifestFilePack; + +// Unpack file pack returned by manifestFilePackGet() +ManifestFile manifestFileUnpack(const ManifestFilePack *filePack); + +// Get file in pack format by index +__attribute__((always_inline)) static inline const ManifestFilePack * +manifestFilePackGet(const Manifest *const this, const unsigned int fileIdx) { - return lstGet(THIS_PUB(Manifest)->fileList, fileIdx); + return *(ManifestFilePack **)lstGet(THIS_PUB(Manifest)->fileList, fileIdx); } -void manifestFileAdd(Manifest *this, const ManifestFile *file); -const ManifestFile *manifestFileFind(const Manifest *this, const String *name); +// Get file name +__attribute__((always_inline)) static inline const String * +manifestFileNameGet(const Manifest *const this, const unsigned int fileIdx) +{ + return (const String *)manifestFilePackGet(this, fileIdx); +} -// If the file requested is not found in the list, return the default passed rather than throw an error -__attribute__((always_inline)) static inline const ManifestFile * -manifestFileFindDefault(const Manifest *const this, const String *const name, const ManifestFile *const fileDefault) +// Get file by index +__attribute__((always_inline)) static inline ManifestFile +manifestFile(const Manifest *const this, const unsigned int fileIdx) +{ + return manifestFileUnpack(manifestFilePackGet(this, fileIdx)); +} + +// Add a file +void manifestFileAdd(Manifest *this, ManifestFile file); + +// Find file in pack format by name +const ManifestFilePack *manifestFilePackFind(const Manifest *this, const String *name); + +// Find file by name +__attribute__((always_inline)) static inline ManifestFile +manifestFileFind(const Manifest *const this, const String *const name) { ASSERT_INLINE(name != NULL); - return lstFindDefault(THIS_PUB(Manifest)->fileList, &name, (void *)fileDefault); + return manifestFileUnpack(manifestFilePackFind(this, name)); +} + +// Does the file exist? +__attribute__((always_inline)) static inline bool +manifestFileExists(const Manifest *const this, const String *const name) +{ + ASSERT_INLINE(name != NULL); + return lstFindDefault(THIS_PUB(Manifest)->fileList, &name, NULL) != NULL; } void manifestFileRemove(const Manifest *this, const String *name); @@ -286,7 +317,7 @@ manifestFileTotal(const Manifest *const this) // Update a file with new data void manifestFileUpdate( Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference, - bool checksumPage, bool checksumPageError, const VariantList *checksumPageErrorList); + bool checksumPage, bool checksumPageError, const String *checksumPageErrorList); /*********************************************************************************************************************************** Link functions and getters/setters diff --git a/test/define.yaml b/test/define.yaml index c99ec09eb..1f6e6b589 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -73,7 +73,7 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: type-convert - total: 11 + total: 12 coverage: - common/type/convert diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index fa5790d32..266213cdb 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -23,7 +23,7 @@ typedef struct TestBackupValidateCallbackData { const Storage *storage; // Storage object when needed (e.g. fileCompressed = true) const String *path; // Subpath when storage is specified - const Manifest *manifest; // Manifest to check for files/links/paths + Manifest *manifest; // Manifest to check for files/links/paths const ManifestData *manifestData; // Manifest data String *content; // String where content should be added } TestBackupValidateCallbackData; @@ -76,30 +76,31 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info) // Check against the manifest // --------------------------------------------------------------------------------------------------------------------- - const ManifestFile *file = manifestFileFind(data->manifest, manifestName); + ManifestFilePack **const filePack = manifestFilePackFindInternal(data->manifest, manifestName); + ManifestFile file = manifestFileUnpack(*filePack); // Test size and repo-size. If compressed then set the repo-size to size so it will not be in test output. Even the same // compression algorithm can give slightly different results based on the version so repo-size is not deterministic for // compression. - if (size != file->size) + if (size != file.size) THROW_FMT(AssertError, "'%s' size does match manifest", strZ(manifestName)); - if (info->size != file->sizeRepo) + if (info->size != file.sizeRepo) THROW_FMT(AssertError, "'%s' repo size does match manifest", strZ(manifestName)); if (data->manifestData->backupOptionCompressType != compressTypeNone) - ((ManifestFile *)file)->sizeRepo = file->size; + file.sizeRepo = file.size; // Test the checksum. pg_control and WAL headers have different checksums depending on cpu architecture so remove // the checksum from the test output. - if (!strEqZ(checksum, file->checksumSha1)) + if (!strEqZ(checksum, file.checksumSha1)) THROW_FMT(AssertError, "'%s' checksum does match manifest", strZ(manifestName)); if (strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL) || strBeginsWith( manifestName, strNewFmt(MANIFEST_TARGET_PGDATA "/%s/", strZ(pgWalPath(data->manifestData->pgVersion))))) { - ((ManifestFile *)file)->checksumSha1[0] = '\0'; + file.checksumSha1[0] = '\0'; } // Test mode, user, group. These values are not in the manifest but we know what they should be based on the default @@ -113,6 +114,9 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info) if (!strEq(info->group, TEST_GROUP_STR)) THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(manifestName)); + // Update changes to manifest file + manifestFilePackUpdate(data->manifest, filePack, &file); + break; } @@ -1310,7 +1314,7 @@ testRun(void) manifestTargetAdd(manifestResume, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")}); manifestPathAdd(manifestResume, &(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR}); - manifestFileAdd(manifestResume, &(ManifestFile){.name = STRDEF("pg_data/" PG_FILE_PGVERSION)}); + manifestFileAdd(manifestResume, (ManifestFile){.name = STRDEF("pg_data/" PG_FILE_PGVERSION)}); manifestSave( manifestResume, @@ -1430,7 +1434,7 @@ testRun(void) OBJ_NEW_BEGIN(Manifest) { manifest = manifestNewInternal(); - manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF("pg_data/test")}); + manifestFileAdd(manifest, (ManifestFile){.name = STRDEF("pg_data/test")}); } OBJ_NEW_END(); @@ -1748,9 +1752,12 @@ testRun(void) storagePg(), PG_FILE_PGVERSION, storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION", strZ(resumeLabel)))); - strcpy( - ((ManifestFile *)manifestFileFind(manifestResume, STRDEF("pg_data/PG_VERSION")))->checksumSha1, - "06d06bb31b570b94d7b4325f511f853dbe771c21"); + ManifestFilePack **const filePack = manifestFilePackFindInternal(manifestResume, STRDEF("pg_data/PG_VERSION")); + ManifestFile file = manifestFileUnpack(*filePack); + + strcpy(file.checksumSha1, "06d06bb31b570b94d7b4325f511f853dbe771c21"); + + manifestFilePackUpdate(manifestResume, filePack, &file); // Save the resume manifest manifestSave( @@ -1841,14 +1848,19 @@ testRun(void) HRN_STORAGE_PUT_EMPTY( storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/global/pg_control.gz", strZ(resumeLabel)))); - ((ManifestFile *)manifestFileFind(manifestResume, STRDEF("pg_data/global/pg_control")))->checksumSha1[0] = 0; + ManifestFilePack **const filePack = manifestFilePackFindInternal(manifestResume, STRDEF("pg_data/global/pg_control")); + ManifestFile file = manifestFileUnpack(*filePack); + + file.checksumSha1[0] = 0; + + manifestFilePackUpdate(manifestResume, filePack, &file); // Size does not match between cluster and resume manifest HRN_STORAGE_PUT_Z(storagePgWrite(), "size-mismatch", "TEST", .timeModified = backupTimeStart); HRN_STORAGE_PUT_EMPTY( storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/size-mismatch.gz", strZ(resumeLabel)))); manifestFileAdd( - manifestResume, &(ManifestFile){ + manifestResume, (ManifestFile){ .name = STRDEF("pg_data/size-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", .size = 33}); @@ -1857,7 +1869,7 @@ testRun(void) HRN_STORAGE_PUT_EMPTY( storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch.gz", strZ(resumeLabel)))); manifestFileAdd( - manifestResume, &(ManifestFile){ + manifestResume, (ManifestFile){ .name = STRDEF("pg_data/time-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", .size = 4, .timestamp = backupTimeStart - 1}); @@ -1867,7 +1879,7 @@ testRun(void) storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/zero-size.gz", strZ(resumeLabel))), "ZERO-SIZE"); manifestFileAdd( - manifestResume, &(ManifestFile){.name = STRDEF("pg_data/zero-size"), .size = 0, .timestamp = backupTimeStart}); + manifestResume, (ManifestFile){.name = STRDEF("pg_data/zero-size"), .size = 0, .timestamp = backupTimeStart}); // Path is not in manifest HRN_STORAGE_PATH_CREATE( @@ -2028,7 +2040,7 @@ testRun(void) HRN_STORAGE_PUT_EMPTY( storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/resume-ref.gz", strZ(resumeLabel)))); manifestFileAdd( - manifestResume, &(ManifestFile){.name = STRDEF("pg_data/resume-ref"), .size = 0, .reference = STRDEF("BOGUS")}); + manifestResume, (ManifestFile){.name = STRDEF("pg_data/resume-ref"), .size = 0, .reference = STRDEF("BOGUS")}); // Time does not match between cluster and resume manifest (but resume because time is in future so delta enabled). Note // also that the repo file is intenionally corrupt to generate a warning about corruption in the repository. @@ -2036,7 +2048,7 @@ testRun(void) HRN_STORAGE_PUT_EMPTY( storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch2.gz", strZ(resumeLabel)))); manifestFileAdd( - manifestResume, &(ManifestFile){ + manifestResume, (ManifestFile){ .name = STRDEF("pg_data/time-mismatch2"), .checksumSha1 = "984816fd329622876e14907634264e6f332e9fb3", .size = 4, .timestamp = backupTimeStart}); diff --git a/test/src/module/command/restoreTest.c b/test/src/module/command/restoreTest.c index 9b7a1e3f6..cf3a66893 100644 --- a/test/src/module/command/restoreTest.c +++ b/test/src/module/command/restoreTest.c @@ -131,7 +131,7 @@ testManifestMinimal(const String *label, unsigned int pgVersion, const String *p manifestPathAdd(result, &pathBase); ManifestFile fileVersion = { .name = STRDEF("pg_data/" PG_FILE_PGVERSION), .mode = 0600, .group = groupName(), .user = userName()}; - manifestFileAdd(result, &fileVersion); + manifestFileAdd(result, fileVersion); } OBJ_NEW_END(); @@ -815,7 +815,7 @@ testRun(void) manifestFileAdd( manifest, - &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/pg_hba.conf"), .size = 4, .timestamp = 1482182860}); + (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/pg_hba.conf"), .size = 4, .timestamp = 1482182860}); TEST_RESULT_VOID(restoreManifestMap(manifest), "remap links"); @@ -982,7 +982,7 @@ testRun(void) ManifestPath path = {.name = STRDEF("pg_data/bogus_path"), .user = STRDEF("path-user-bogus")}; manifestPathAdd(manifest, &path); ManifestFile file = {.name = STRDEF("pg_data/bogus_file"), .mode = 0600, .group = STRDEF("file-group-bogus")}; - manifestFileAdd(manifest, &file); + manifestFileAdd(manifest, file); ManifestLink link = {.name = STRDEF("pg_data/bogus_link"), .destination = STRDEF("/"), .group = STRDEF("link-group-bogus")}; manifestLinkAdd(manifest, &link); @@ -1034,7 +1034,7 @@ testRun(void) manifest = testManifestMinimal(STRDEF("20161219-212741F_20161219-21275D"), PG_VERSION_96, pgPath); - manifestFileAdd(manifest, &file); + manifestFileAdd(manifest, file); manifestLinkAdd(manifest, &link); TEST_RESULT_VOID(restoreManifestOwner(manifest, &rootReplaceUser, &rootReplaceGroup), "check ownership"); @@ -1051,7 +1051,7 @@ testRun(void) TEST_TITLE("owner is root and ownership of pg_data is bad"); manifestPathAdd(manifest, &path); - manifestFileAdd(manifest, &file); + manifestFileAdd(manifest, file); HRN_SYSTEM_FMT("sudo chown 77777:77777 %s", strZ(pgPath)); @@ -1243,7 +1243,7 @@ testRun(void) HRN_SYSTEM_FMT("rm -rf %s/*", strZ(pgPath)); - manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)}); + manifestFileAdd(manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)}); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_POSTGRESQLAUTOCONF); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_RECOVERYSIGNAL); @@ -1303,7 +1303,7 @@ testRun(void) manifest->pub.data.pgCatalogVersion = hrnPgCatalogVersion(PG_VERSION_90); manifestTargetAdd(manifest, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")}); - manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION)}); + manifestFileAdd(manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION)}); } OBJ_NEW_END(); @@ -1324,11 +1324,11 @@ testRun(void) manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("user-made-system-db"), .id = 16380, .lastSystemId = 12168}); manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF(UTF8_DB_NAME), .id = 16384, .lastSystemId = 12168}); manifestFileAdd( - manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/1/" PG_FILE_PGVERSION)}); + manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/1/" PG_FILE_PGVERSION)}); manifestFileAdd( - manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16381/" PG_FILE_PGVERSION)}); + manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16381/" PG_FILE_PGVERSION)}); manifestFileAdd( - manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16385/" PG_FILE_PGVERSION)}); + manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16385/" PG_FILE_PGVERSION)}); } MEM_CONTEXT_END(); @@ -1353,7 +1353,7 @@ testRun(void) MEM_CONTEXT_BEGIN(manifest->pub.memContext) { manifestFileAdd( - manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16384/" PG_FILE_PGVERSION)}); + manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16384/" PG_FILE_PGVERSION)}); } MEM_CONTEXT_END(); @@ -1424,7 +1424,7 @@ testRun(void) { manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test2"), .id = 32768, .lastSystemId = 12168}); manifestFileAdd( - manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)}); + manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)}); } MEM_CONTEXT_END(); @@ -1448,7 +1448,7 @@ testRun(void) .name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387"), .tablespaceId = 16387, .tablespaceName = STRDEF("ts1"), .path = STRDEF("/ts1")}); manifestFileAdd( - manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)}); + manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)}); } MEM_CONTEXT_END(); @@ -1470,7 +1470,7 @@ testRun(void) { manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test3"), .id = 65536, .lastSystemId = 12168}); manifestFileAdd( - manifest, &(ManifestFile){ + manifest, (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387/PG_9.4_201409291/65536/" PG_FILE_PGVERSION)}); } MEM_CONTEXT_END(); @@ -2053,7 +2053,7 @@ testRun(void) // PG_VERSION manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"}); @@ -2184,7 +2184,7 @@ testRun(void) // tablespace_map (will be ignored during restore) manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO}); HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP); @@ -2218,7 +2218,7 @@ testRun(void) // pg_tblspc/1/16384/PG_VERSION manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/1/16384/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"}); @@ -2400,7 +2400,7 @@ testRun(void) manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL), .size = 8192, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "5e2b96c19c4f5c63a5afa2de504d29fe64a4c908"}); @@ -2409,7 +2409,7 @@ testRun(void) // global/999 manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/999"), .size = 0, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO, .reference = STRDEF(TEST_LABEL)}); @@ -2418,7 +2418,7 @@ testRun(void) // PG_VERSION manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"}); @@ -2439,7 +2439,7 @@ testRun(void) // base/1/PG_VERSION manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA "base/1/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"}); @@ -2452,7 +2452,7 @@ testRun(void) manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA "base/1/2"), .size = 8192, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "4d7b2a36c5387decf799352a3751883b7ceb96aa"}); @@ -2470,7 +2470,7 @@ testRun(void) // base/16384/PG_VERSION manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA "base/16384/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"}); @@ -2483,7 +2483,7 @@ testRun(void) manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA "base/16384/16385"), .size = 16384, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "d74e5f7ebe52a3ed468ba08c5b6aefaccd1ca88f"}); @@ -2499,7 +2499,7 @@ testRun(void) // base/32768/PG_VERSION manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA "base/32768/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"}); @@ -2512,7 +2512,7 @@ testRun(void) manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA "base/32768/32769"), .size = 32768, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "a40f0986acb1531ce0cc75a23dcf8aa406ae9081"}); @@ -2529,7 +2529,7 @@ testRun(void) .name = name, .destination = STRDEF("../config/postgresql.conf"), .group = groupName(), .user = userName()}); manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA "postgresql.conf"), .size = 15, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "98b8abb2e681e2a5a7d8ab082c0a79727887558d"}); @@ -2546,7 +2546,7 @@ testRun(void) .name = name, .destination = STRDEF("../config/pg_hba.conf"), .group = groupName(), .user = userName()}); manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA "pg_hba.conf"), .size = 11, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = "401215e092779574988a854d8c7caed7f91dba4b"}); @@ -2555,7 +2555,7 @@ testRun(void) // tablespace_map (will be ignored during restore) manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO}); HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP); diff --git a/test/src/module/common/typeConvertTest.c b/test/src/module/common/typeConvertTest.c index fc5fcb0a0..27f6184ec 100644 --- a/test/src/module/common/typeConvertTest.c +++ b/test/src/module/common/typeConvertTest.c @@ -202,5 +202,36 @@ testRun(void) TEST_RESULT_UINT(cvtZToUInt64("18446744073709551615"), 18446744073709551615U, "convert string to uint64"); } + // ***************************************************************************************************************************** + if (testBegin("cvtUInt64ToVarInt128() and cvtUInt64FromVarInt128()")) + { + uint8_t buffer[CVT_VARINT128_BUFFER_SIZE]; + size_t bufferPos; + + bufferPos = 0; + TEST_ERROR(cvtUInt64ToVarInt128(9999, buffer, &bufferPos, 1), AssertError, "buffer overflow"); + + bufferPos = 0; + TEST_RESULT_VOID(cvtUInt64ToVarInt128(0xFFFFFFFFFFFFFFFF, buffer, &bufferPos, sizeof(buffer)), "to varint-128"); + TEST_RESULT_UINT(bufferPos, 10, "check buffer position"); + + bufferPos = 0; + TEST_RESULT_UINT(cvtUInt64FromVarInt128(buffer, &bufferPos), 0xFFFFFFFFFFFFFFFF, "to uint64"); + TEST_RESULT_UINT(bufferPos, 10, "check buffer position"); + + bufferPos = 0; + TEST_RESULT_VOID(cvtUInt64ToVarInt128(0, buffer, &bufferPos, sizeof(buffer)), "to varint-128"); + TEST_RESULT_UINT(bufferPos, 1, "check buffer position"); + + bufferPos = 0; + TEST_RESULT_UINT(cvtUInt64FromVarInt128(buffer, &bufferPos), 0, "to uint64"); + TEST_RESULT_UINT(bufferPos, 1, "check buffer position"); + + uint8_t buffer2[CVT_VARINT128_BUFFER_SIZE + 1]; + memset(buffer2, 0xFF, sizeof(buffer2)); + bufferPos = 0; + TEST_ERROR(cvtUInt64FromVarInt128(buffer2, &bufferPos), FormatError, "unterminated varint-128 integer"); + } + FUNCTION_HARNESS_RETURN_VOID(); } diff --git a/test/src/module/info/manifestTest.c b/test/src/module/info/manifestTest.c index 50d1c51cb..d15d98448 100644 --- a/test/src/module/info/manifestTest.c +++ b/test/src/module/info/manifestTest.c @@ -950,7 +950,7 @@ testRun(void) manifestFileAdd( manifest, - &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860}); + (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860}); TEST_RESULT_VOID(manifestBuildValidate(manifest, false, 1482182860, false), "validate manifest"); TEST_RESULT_INT(manifest->pub.data.backupTimestampCopyStart, 1482182860, "check copy start"); @@ -1029,22 +1029,22 @@ testRun(void) &(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR, .mode = 0700, .group = STRDEF("test"), .user = STRDEF("test")}); manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/BOGUS"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182861, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); } @@ -1059,17 +1059,17 @@ testRun(void) manifestFileAdd( manifestPrior, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860, .checksumSha1 = "da39a3ee5e6b4b0d3255bfef95601890afd80709"}); manifestFileAdd( manifestPrior, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182860, .checksumSha1 = "ccccccccccaaaaaaaaaabbbbbbbbbbdddddddddd"}); manifestFileAdd( manifestPrior, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"}); } @@ -1109,18 +1109,18 @@ testRun(void) lstClear(manifest->pub.fileList); manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); manifestFileAdd( manifestPrior, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .reference = STRDEF("20190101-010101F_20190202-010101D"), .checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"}); @@ -1160,7 +1160,7 @@ testRun(void) manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182859, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); @@ -1171,11 +1171,11 @@ testRun(void) varLstAdd(checksumPageErrorList, varNewUInt(77)); manifestFileAdd( manifestPrior, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .reference = STRDEF("20190101-010101F_20190202-010101D"), .checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", .checksumPage = true, .checksumPageError = true, - .checksumPageErrorList = checksumPageErrorList}); + .checksumPageErrorList = jsonFromVar(varNewVarLst(checksumPageErrorList))}); TEST_RESULT_VOID(manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, NULL), "incremental manifest"); @@ -1211,18 +1211,18 @@ testRun(void) lstClear(manifest->pub.fileList); manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); manifestFileAdd( manifestPrior, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .reference = STRDEF("20190101-010101F_20190202-010101D"), .checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"}); @@ -1279,14 +1279,14 @@ testRun(void) lstClear(manifest->pub.fileList); manifestFileAdd( manifest, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861, .mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")}); manifest->pub.data.backupOptionOnline = BOOL_TRUE_VAR; manifestFileAdd( manifestPrior, - &(ManifestFile){ + (ManifestFile){ .name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"}); @@ -1729,18 +1729,14 @@ testRun(void) TEST_TITLE("manifest getters"); // ManifestFile getters - const ManifestFile *file = NULL; + ManifestFile file = {0}; TEST_ERROR(manifestFileFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest file list"); TEST_ASSIGN(file, manifestFileFind(manifest, STRDEF("pg_data/PG_VERSION")), "manifestFileFind()"); - TEST_RESULT_STR_Z(file->name, "pg_data/PG_VERSION", "find file"); + TEST_RESULT_STR_Z(file.name, "pg_data/PG_VERSION", "find file"); TEST_RESULT_STR_Z( - manifestFileFindDefault(manifest, STRDEF("bogus"), file)->name, "pg_data/PG_VERSION", - "manifestFileFindDefault() - return default"); - TEST_RESULT_STR_Z( - manifestFileFind(manifest, STRDEF("pg_data/special-@#!$^&*()_+~`{}[]\\:;"))->name, + manifestFileFind(manifest, STRDEF("pg_data/special-@#!$^&*()_+~`{}[]\\:;")).name, "pg_data/special-@#!$^&*()_+~`{}[]\\:;", "find special file"); - TEST_ASSIGN(file, manifestFileFindDefault(manifest, STRDEF("bogus"), NULL), "manifestFileFindDefault()"); - TEST_RESULT_PTR(file, NULL, "return default NULL"); + TEST_RESULT_BOOL(manifestFileExists(manifest, STRDEF("bogus")), false, "manifest file does not exist"); TEST_RESULT_VOID( manifestFileUpdate(manifest, STRDEF("pg_data/postgresql.conf"), 4457, 4457, "", NULL, false, false, NULL), diff --git a/test/src/module/performance/typeTest.c b/test/src/module/performance/typeTest.c index f096f9609..ff4c24bc2 100644 --- a/test/src/module/performance/typeTest.c +++ b/test/src/module/performance/typeTest.c @@ -317,8 +317,8 @@ testRun(void) for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) { - const ManifestFile *file = manifestFile(manifest, fileIdx); - ASSERT(file == manifestFileFind(manifest, file->name)); + const ManifestFile file = manifestFile(manifest, fileIdx); + ASSERT(strEq(file.name, manifestFileFind(manifest, file.name).name)); } TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));