You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-07-03 00:26:59 +02:00
Pack manifest file structs to save memory.
Manifests with a very large number of files can use a considerable amount of memory. There are a lot of zeroes in the data so it can be stored more efficiently by using base-128 varint encoding for the integers and storing the strings in the same allocation. The downside is that the data needs to be unpacked in order to be used, but in most cases this seems fast enough (about 10% slower than before) except for saving the manifest, which is 10% slower up to 10 million files and then gets about 5x slower by 100 million (two minutes on my M1 Mac). Profiling does not show this slowdown so I wonder if this is related to the change in memory layout. Curiously, the function that increased most was jsonFromStrInternal(), which was not modified. That gives more weight to the idea that there is some kind of memory issue going on here and one hopes that servers would be less affected. Either way, they largest use cases we have seen are for about 6 million files so if we can improve that case I believe we will be better off. Further analysis showed that most of the time was taken up writing the size and timestamp fields, which makes almost no sense. The same amount of time was used if they were hard-coded to 0, which points to some odd memory issue on the M1 architecture. This change has been planned for a while, but the particular impetus at this time is that small file support requires additional fields that would increase manifest memory usage by about 20%, even if the feature is not used. Note that the Pack code has been updated to use the new varint encoder, but the decoder remains separate because it needs to fetch one byte at a time.
This commit is contained in:
@ -68,6 +68,7 @@
|
|||||||
<commit subject="Simplify base path mode in mock/all integration tests."/>
|
<commit subject="Simplify base path mode in mock/all integration tests."/>
|
||||||
<commit subject="Simplify manifest defaults."/>
|
<commit subject="Simplify manifest defaults."/>
|
||||||
<commit subject="Convert varNewUInt64() to VARUINT64() where possible in manifest."/>
|
<commit subject="Convert varNewUInt64() to VARUINT64() where possible in manifest."/>
|
||||||
|
<commit subject="Pack manifest file structs to save memory."/>
|
||||||
|
|
||||||
<release-item-contributor-list>
|
<release-item-contributor-list>
|
||||||
<release-item-contributor id="david.steele"/>
|
<release-item-contributor id="david.steele"/>
|
||||||
|
@ -576,37 +576,43 @@ void backupResumeCallback(void *data, const StorageInfo *info)
|
|||||||
if (fileCompressType != compressTypeNone)
|
if (fileCompressType != compressTypeNone)
|
||||||
manifestName = compressExtStrip(manifestName, fileCompressType);
|
manifestName = compressExtStrip(manifestName, fileCompressType);
|
||||||
|
|
||||||
// Find the file in both manifests
|
|
||||||
const ManifestFile *file = manifestFileFindDefault(resumeData->manifest, manifestName, NULL);
|
|
||||||
const ManifestFile *fileResume = manifestFileFindDefault(resumeData->manifestResume, manifestName, NULL);
|
|
||||||
|
|
||||||
// Check if the file can be resumed or must be removed
|
// Check if the file can be resumed or must be removed
|
||||||
const char *removeReason = NULL;
|
const char *removeReason = NULL;
|
||||||
|
|
||||||
if (fileCompressType != resumeData->compressType)
|
if (fileCompressType != resumeData->compressType)
|
||||||
removeReason = "mismatched compression type";
|
removeReason = "mismatched compression type";
|
||||||
else if (file == NULL)
|
else if (!manifestFileExists(resumeData->manifest, manifestName))
|
||||||
removeReason = "missing in manifest";
|
removeReason = "missing in manifest";
|
||||||
else if (file->reference != NULL)
|
else
|
||||||
|
{
|
||||||
|
const ManifestFile file = manifestFileFind(resumeData->manifest, manifestName);
|
||||||
|
|
||||||
|
if (file.reference != NULL)
|
||||||
removeReason = "reference in manifest";
|
removeReason = "reference in manifest";
|
||||||
else if (fileResume == NULL)
|
else if (!manifestFileExists(resumeData->manifestResume, manifestName))
|
||||||
removeReason = "missing in resumed manifest";
|
removeReason = "missing in resumed manifest";
|
||||||
else if (fileResume->reference != NULL)
|
else
|
||||||
|
{
|
||||||
|
const ManifestFile fileResume = manifestFileFind(resumeData->manifestResume, manifestName);
|
||||||
|
|
||||||
|
if (fileResume.reference != NULL)
|
||||||
removeReason = "reference in resumed manifest";
|
removeReason = "reference in resumed manifest";
|
||||||
else if (fileResume->checksumSha1[0] == '\0')
|
else if (fileResume.checksumSha1[0] == '\0')
|
||||||
removeReason = "no checksum in resumed manifest";
|
removeReason = "no checksum in resumed manifest";
|
||||||
else if (file->size != fileResume->size)
|
else if (file.size != fileResume.size)
|
||||||
removeReason = "mismatched size";
|
removeReason = "mismatched size";
|
||||||
else if (!resumeData->delta && file->timestamp != fileResume->timestamp)
|
else if (!resumeData->delta && file.timestamp != fileResume.timestamp)
|
||||||
removeReason = "mismatched timestamp";
|
removeReason = "mismatched timestamp";
|
||||||
else if (file->size == 0)
|
else if (file.size == 0)
|
||||||
// ??? don't resume zero size files because Perl wouldn't -- this can be removed after the migration)
|
// ??? don't resume zero size files because Perl wouldn't -- this can be removed after the migration)
|
||||||
removeReason = "zero size";
|
removeReason = "zero size";
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
manifestFileUpdate(
|
manifestFileUpdate(
|
||||||
resumeData->manifest, manifestName, file->size, fileResume->sizeRepo, fileResume->checksumSha1, NULL,
|
resumeData->manifest, manifestName, file.size, fileResume.sizeRepo, fileResume.checksumSha1, NULL,
|
||||||
fileResume->checksumPage, fileResume->checksumPageError, fileResume->checksumPageErrorList);
|
fileResume.checksumPage, fileResume.checksumPageError, fileResume.checksumPageErrorList);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove the file if it could not be resumed
|
// Remove the file if it could not be resumed
|
||||||
@ -981,7 +987,7 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti
|
|||||||
file.checksumSha1, strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))),
|
file.checksumSha1, strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))),
|
||||||
HASH_TYPE_SHA1_SIZE_HEX + 1);
|
HASH_TYPE_SHA1_SIZE_HEX + 1);
|
||||||
|
|
||||||
manifestFileAdd(manifest, &file);
|
manifestFileAdd(manifest, file);
|
||||||
|
|
||||||
LOG_DETAIL_FMT("wrote '%s' file returned from pg_stop_backup()", strZ(name));
|
LOG_DETAIL_FMT("wrote '%s' file returned from pg_stop_backup()", strZ(name));
|
||||||
}
|
}
|
||||||
@ -1165,7 +1171,7 @@ backupJobResult(
|
|||||||
{
|
{
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
const ManifestFile *const file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
|
const ManifestFile file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
|
||||||
const unsigned int processId = protocolParallelJobProcessId(job);
|
const unsigned int processId = protocolParallelJobProcessId(job);
|
||||||
|
|
||||||
PackRead *const jobResult = protocolParallelJobResult(job);
|
PackRead *const jobResult = protocolParallelJobResult(job);
|
||||||
@ -1205,7 +1211,7 @@ backupJobResult(
|
|||||||
else if (copyResult == backupCopyResultSkip)
|
else if (copyResult == backupCopyResultSkip)
|
||||||
{
|
{
|
||||||
LOG_DETAIL_PID_FMT(processId, "skip file removed by database %s", strZ(fileLog));
|
LOG_DETAIL_PID_FMT(processId, "skip file removed by database %s", strZ(fileLog));
|
||||||
strLstAdd(fileRemove, file->name);
|
strLstAdd(fileRemove, file.name);
|
||||||
}
|
}
|
||||||
// Else file was copied so update manifest
|
// Else file was copied so update manifest
|
||||||
else
|
else
|
||||||
@ -1220,13 +1226,13 @@ backupJobResult(
|
|||||||
" continue but this may be an issue unless the resumed backup path in the repository is known to be"
|
" continue but this may be an issue unless the resumed backup path in the repository is known to be"
|
||||||
" corrupted.\n"
|
" corrupted.\n"
|
||||||
"NOTE: this does not indicate a problem with the PostgreSQL page checksums.",
|
"NOTE: this does not indicate a problem with the PostgreSQL page checksums.",
|
||||||
strZ(file->name), file->checksumSha1);
|
strZ(file.name), file.checksumSha1);
|
||||||
}
|
}
|
||||||
|
|
||||||
LOG_DETAIL_PID_FMT(processId, "backup file %s (%s)%s", strZ(fileLog), strZ(logProgress), strZ(logChecksum));
|
LOG_DETAIL_PID_FMT(processId, "backup file %s (%s)%s", strZ(fileLog), strZ(logProgress), strZ(logChecksum));
|
||||||
|
|
||||||
// If the file had page checksums calculated during the copy
|
// If the file had page checksums calculated during the copy
|
||||||
ASSERT((!file->checksumPage && checksumPageResult == NULL) || (file->checksumPage && checksumPageResult != NULL));
|
ASSERT((!file.checksumPage && checksumPageResult == NULL) || (file.checksumPage && checksumPageResult != NULL));
|
||||||
|
|
||||||
bool checksumPageError = false;
|
bool checksumPageError = false;
|
||||||
const VariantList *checksumPageErrorList = NULL;
|
const VariantList *checksumPageErrorList = NULL;
|
||||||
@ -1300,8 +1306,8 @@ backupJobResult(
|
|||||||
|
|
||||||
// Update file info and remove any reference to the file's existence in a prior backup
|
// Update file info and remove any reference to the file's existence in a prior backup
|
||||||
manifestFileUpdate(
|
manifestFileUpdate(
|
||||||
manifest, file->name, copySize, repoSize, strZ(copyChecksum), VARSTR(NULL), file->checksumPage,
|
manifest, file.name, copySize, repoSize, strZ(copyChecksum), VARSTR(NULL), file.checksumPage,
|
||||||
checksumPageError, checksumPageErrorList);
|
checksumPageError, checksumPageErrorList != NULL ? jsonFromVar(varNewVarLst(checksumPageErrorList)) : NULL);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
@ -1420,14 +1426,18 @@ backupProcessQueueComparator(const void *item1, const void *item2)
|
|||||||
ASSERT(item1 != NULL);
|
ASSERT(item1 != NULL);
|
||||||
ASSERT(item2 != NULL);
|
ASSERT(item2 != NULL);
|
||||||
|
|
||||||
|
// Unpack files
|
||||||
|
ManifestFile file1 = manifestFileUnpack(*(const ManifestFilePack **)item1);
|
||||||
|
ManifestFile file2 = manifestFileUnpack(*(const ManifestFilePack **)item2);
|
||||||
|
|
||||||
// If the size differs then that's enough to determine order
|
// If the size differs then that's enough to determine order
|
||||||
if ((*(ManifestFile **)item1)->size < (*(ManifestFile **)item2)->size)
|
if (file1.size < file2.size)
|
||||||
FUNCTION_TEST_RETURN(-1);
|
FUNCTION_TEST_RETURN(-1);
|
||||||
else if ((*(ManifestFile **)item1)->size > (*(ManifestFile **)item2)->size)
|
else if (file1.size > file2.size)
|
||||||
FUNCTION_TEST_RETURN(1);
|
FUNCTION_TEST_RETURN(1);
|
||||||
|
|
||||||
// If size is the same then use name to generate a deterministic ordering (names must be unique)
|
// If size is the same then use name to generate a deterministic ordering (names must be unique)
|
||||||
FUNCTION_TEST_RETURN(strCmp((*(ManifestFile **)item1)->name, (*(ManifestFile **)item2)->name));
|
FUNCTION_TEST_RETURN(strCmp(file1.name, file2.name));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper to generate the backup queues
|
// Helper to generate the backup queues
|
||||||
@ -1479,20 +1489,21 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData)
|
|||||||
|
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFile(manifest, fileIdx);
|
const ManifestFilePack *const filePack = manifestFilePackGet(manifest, fileIdx);
|
||||||
|
const ManifestFile file = manifestFileUnpack(filePack);
|
||||||
|
|
||||||
// If the file is a reference it should only be backed up if delta and not zero size
|
// If the file is a reference it should only be backed up if delta and not zero size
|
||||||
if (file->reference != NULL && (!jobData->delta || file->size == 0))
|
if (file.reference != NULL && (!jobData->delta || file.size == 0))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
// Is pg_control in the backup?
|
// Is pg_control in the backup?
|
||||||
if (strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)))
|
if (strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)))
|
||||||
pgControlFound = true;
|
pgControlFound = true;
|
||||||
|
|
||||||
// Files that must be copied from the primary are always put in queue 0 when backup from standby
|
// Files that must be copied from the primary are always put in queue 0 when backup from standby
|
||||||
if (jobData->backupStandby && backupProcessFilePrimary(jobData->standbyExp, file->name))
|
if (jobData->backupStandby && backupProcessFilePrimary(jobData->standbyExp, file.name))
|
||||||
{
|
{
|
||||||
lstAdd(*(List **)lstGet(jobData->queueList, 0), &file);
|
lstAdd(*(List **)lstGet(jobData->queueList, 0), &filePack);
|
||||||
}
|
}
|
||||||
// Else find the correct queue by matching the file to a target
|
// Else find the correct queue by matching the file to a target
|
||||||
else
|
else
|
||||||
@ -1504,7 +1515,7 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData)
|
|||||||
{
|
{
|
||||||
CHECK(AssertError, targetIdx < strLstSize(targetList), "backup target not found");
|
CHECK(AssertError, targetIdx < strLstSize(targetList), "backup target not found");
|
||||||
|
|
||||||
if (strBeginsWith(file->name, strLstGet(targetList, targetIdx)))
|
if (strBeginsWith(file.name, strLstGet(targetList, targetIdx)))
|
||||||
break;
|
break;
|
||||||
|
|
||||||
targetIdx++;
|
targetIdx++;
|
||||||
@ -1512,11 +1523,11 @@ backupProcessQueue(Manifest *const manifest, BackupJobData *const jobData)
|
|||||||
while (1);
|
while (1);
|
||||||
|
|
||||||
// Add file to queue
|
// Add file to queue
|
||||||
lstAdd(*(List **)lstGet(jobData->queueList, targetIdx + queueOffset), &file);
|
lstAdd(*(List **)lstGet(jobData->queueList, targetIdx + queueOffset), &filePack);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add size to total
|
// Add size to total
|
||||||
result += file->size;
|
result += file.size;
|
||||||
|
|
||||||
// Increment total files
|
// Increment total files
|
||||||
fileTotal++;
|
fileTotal++;
|
||||||
@ -1600,21 +1611,21 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
|
|||||||
|
|
||||||
if (!lstEmpty(queue))
|
if (!lstEmpty(queue))
|
||||||
{
|
{
|
||||||
const ManifestFile *file = *(ManifestFile **)lstGet(queue, 0);
|
const ManifestFile file = manifestFileUnpack(*(ManifestFilePack **)lstGet(queue, 0));
|
||||||
|
|
||||||
// Create backup job
|
// Create backup job
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE);
|
||||||
PackWrite *const param = protocolCommandParam(command);
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
|
|
||||||
pckWriteStrP(param, manifestPathPg(file->name));
|
pckWriteStrP(param, manifestPathPg(file.name));
|
||||||
pckWriteBoolP(param, !strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)));
|
pckWriteBoolP(param, !strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)));
|
||||||
pckWriteU64P(param, file->size);
|
pckWriteU64P(param, file.size);
|
||||||
pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file->name));
|
pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file.name));
|
||||||
pckWriteStrP(param, file->checksumSha1[0] != 0 ? STR(file->checksumSha1) : NULL);
|
pckWriteStrP(param, file.checksumSha1[0] != 0 ? STR(file.checksumSha1) : NULL);
|
||||||
pckWriteBoolP(param, file->checksumPage);
|
pckWriteBoolP(param, file.checksumPage);
|
||||||
pckWriteU64P(param, jobData->lsnStart);
|
pckWriteU64P(param, jobData->lsnStart);
|
||||||
pckWriteStrP(param, file->name);
|
pckWriteStrP(param, file.name);
|
||||||
pckWriteBoolP(param, file->reference != NULL);
|
pckWriteBoolP(param, file.reference != NULL);
|
||||||
pckWriteU32P(param, jobData->compressType);
|
pckWriteU32P(param, jobData->compressType);
|
||||||
pckWriteI32P(param, jobData->compressLevel);
|
pckWriteI32P(param, jobData->compressLevel);
|
||||||
pckWriteStrP(param, jobData->backupLabel);
|
pckWriteStrP(param, jobData->backupLabel);
|
||||||
@ -1628,7 +1639,7 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
|
|||||||
// Assign job to result
|
// Assign job to result
|
||||||
MEM_CONTEXT_PRIOR_BEGIN()
|
MEM_CONTEXT_PRIOR_BEGIN()
|
||||||
{
|
{
|
||||||
result = protocolParallelJobNew(VARSTR(file->name), command);
|
result = protocolParallelJobNew(VARSTR(file.name), command);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_PRIOR_END();
|
MEM_CONTEXT_PRIOR_END();
|
||||||
|
|
||||||
@ -1774,7 +1785,7 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart
|
|||||||
backupStandby && protocolParallelJobProcessId(job) > 1 ? backupData->hostStandby : backupData->hostPrimary,
|
backupStandby && protocolParallelJobProcessId(job) > 1 ? backupData->hostStandby : backupData->hostPrimary,
|
||||||
storagePathP(
|
storagePathP(
|
||||||
protocolParallelJobProcessId(job) > 1 ? storagePgIdx(pgIdx) : backupData->storagePrimary,
|
protocolParallelJobProcessId(job) > 1 ? storagePgIdx(pgIdx) : backupData->storagePrimary,
|
||||||
manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job)))->name)), fileRemove, job,
|
manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job))).name)), fileRemove, job,
|
||||||
sizeTotal, &sizeProgress);
|
sizeTotal, &sizeProgress);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1814,22 +1825,22 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart
|
|||||||
|
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *const file = manifestFile(manifest, fileIdx);
|
const ManifestFile file = manifestFile(manifest, fileIdx);
|
||||||
|
|
||||||
// If the file has a reference, then it was not copied since it can be retrieved from the referenced backup. However,
|
// If the file has a reference, then it was not copied since it can be retrieved from the referenced backup. However,
|
||||||
// if hardlinking is enabled the link will need to be created.
|
// if hardlinking is enabled the link will need to be created.
|
||||||
if (file->reference != NULL)
|
if (file.reference != NULL)
|
||||||
{
|
{
|
||||||
// If hardlinking is enabled then create a hardlink for files that have not changed since the last backup
|
// If hardlinking is enabled then create a hardlink for files that have not changed since the last backup
|
||||||
if (hardLink)
|
if (hardLink)
|
||||||
{
|
{
|
||||||
LOG_DETAIL_FMT("hardlink %s to %s", strZ(file->name), strZ(file->reference));
|
LOG_DETAIL_FMT("hardlink %s to %s", strZ(file.name), strZ(file.reference));
|
||||||
|
|
||||||
const String *const linkName = storagePathP(
|
const String *const linkName = storagePathP(
|
||||||
storageRepo(), strNewFmt("%s/%s%s", strZ(backupPathExp), strZ(file->name), compressExt));
|
storageRepo(), strNewFmt("%s/%s%s", strZ(backupPathExp), strZ(file.name), compressExt));
|
||||||
const String *const linkDestination = storagePathP(
|
const String *const linkDestination = storagePathP(
|
||||||
storageRepo(),
|
storageRepo(),
|
||||||
strNewFmt(STORAGE_REPO_BACKUP "/%s/%s%s", strZ(file->reference), strZ(file->name), compressExt));
|
strNewFmt(STORAGE_REPO_BACKUP "/%s/%s%s", strZ(file.reference), strZ(file.name), compressExt));
|
||||||
|
|
||||||
THROW_ON_SYS_ERROR_FMT(
|
THROW_ON_SYS_ERROR_FMT(
|
||||||
link(strZ(linkDestination), strZ(linkName)) == -1, FileOpenError,
|
link(strZ(linkDestination), strZ(linkName)) == -1, FileOpenError,
|
||||||
@ -1838,7 +1849,7 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart
|
|||||||
// Else log the reference. With delta, it is possible that references may have been removed if a file needed to be
|
// Else log the reference. With delta, it is possible that references may have been removed if a file needed to be
|
||||||
// recopied.
|
// recopied.
|
||||||
else
|
else
|
||||||
LOG_DETAIL_FMT("reference %s to %s", strZ(file->name), strZ(file->reference));
|
LOG_DETAIL_FMT("reference %s to %s", strZ(file.name), strZ(file.reference));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1974,7 +1985,7 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif
|
|||||||
|
|
||||||
memcpy(file.checksumSha1, strZ(strSubN(archiveFile, 25, 40)), HASH_TYPE_SHA1_SIZE_HEX + 1);
|
memcpy(file.checksumSha1, strZ(strSubN(archiveFile, 25, 40)), HASH_TYPE_SHA1_SIZE_HEX + 1);
|
||||||
|
|
||||||
manifestFileAdd(manifest, &file);
|
manifestFileAdd(manifest, file);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
@ -533,10 +533,10 @@ backupListAdd(
|
|||||||
|
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(repoData->manifest); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(repoData->manifest); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFile(repoData->manifest, fileIdx);
|
const ManifestFile file = manifestFile(repoData->manifest, fileIdx);
|
||||||
|
|
||||||
if (file->checksumPageError)
|
if (file.checksumPageError)
|
||||||
varLstAdd(checksumPageErrorList, varNewStr(manifestPathPg(file->name)));
|
varLstAdd(checksumPageErrorList, varNewStr(manifestPathPg(file.name)));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!varLstEmpty(checksumPageErrorList))
|
if (!varLstEmpty(checksumPageErrorList))
|
||||||
|
@ -533,22 +533,22 @@ restoreManifestMap(Manifest *manifest)
|
|||||||
if (target.name == NULL)
|
if (target.name == NULL)
|
||||||
{
|
{
|
||||||
// Is the specified link a file or a path? Error if they both match.
|
// Is the specified link a file or a path? Error if they both match.
|
||||||
const ManifestPath *const path = manifestPathFindDefault(manifest, manifestName, NULL);
|
const bool pathExists = manifestPathFindDefault(manifest, manifestName, NULL) != NULL;
|
||||||
const ManifestFile *const file = manifestFileFindDefault(manifest, manifestName, NULL);
|
const bool fileExists = manifestFileExists(manifest, manifestName);
|
||||||
|
|
||||||
CHECK(FormatError, path == NULL || file == NULL, "link may not be both file and path");
|
CHECK(FormatError, !pathExists || !fileExists, "link may not be both file and path");
|
||||||
|
|
||||||
target = (ManifestTarget){.name = manifestName, .path = linkPath, .type = manifestTargetTypeLink};
|
target = (ManifestTarget){.name = manifestName, .path = linkPath, .type = manifestTargetTypeLink};
|
||||||
|
|
||||||
// If a file
|
// If a file
|
||||||
if (file != NULL)
|
if (fileExists)
|
||||||
{
|
{
|
||||||
// File needs to be set so the file/path is updated later but set it to something invalid just in case it
|
// File needs to be set so the file/path is updated later but set it to something invalid just in case it
|
||||||
// it does not get updated due to a regression
|
// it does not get updated due to a regression
|
||||||
target.file = DOT_STR;
|
target.file = DOT_STR;
|
||||||
}
|
}
|
||||||
// Else error if not a path
|
// Else error if not a path
|
||||||
else if (path == NULL)
|
else if (!pathExists)
|
||||||
{
|
{
|
||||||
THROW_FMT(
|
THROW_FMT(
|
||||||
LinkMapError,
|
LinkMapError,
|
||||||
@ -666,20 +666,20 @@ restoreManifestOwnerReplace(const String *const owner, const String *const owner
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Helper to get list of owners from a file/link/path list
|
// Helper to get list of owners from a file/link/path list
|
||||||
#define RESTORE_MANIFEST_OWNER_GET(type) \
|
#define RESTORE_MANIFEST_OWNER_GET(type, deref) \
|
||||||
for (unsigned int itemIdx = 0; itemIdx < manifest##type##Total(manifest); itemIdx++) \
|
for (unsigned int itemIdx = 0; itemIdx < manifest##type##Total(manifest); itemIdx++) \
|
||||||
{ \
|
{ \
|
||||||
Manifest##type *item = (Manifest##type *)manifest##type(manifest, itemIdx); \
|
Manifest##type item = deref manifest##type(manifest, itemIdx); \
|
||||||
\
|
\
|
||||||
if (item->user == NULL) \
|
if (item.user == NULL) \
|
||||||
userNull = true; \
|
userNull = true; \
|
||||||
else \
|
else \
|
||||||
strLstAddIfMissing(userList, item->user); \
|
strLstAddIfMissing(userList, item.user); \
|
||||||
\
|
\
|
||||||
if (item->group == NULL) \
|
if (item.group == NULL) \
|
||||||
groupNull = true; \
|
groupNull = true; \
|
||||||
else \
|
else \
|
||||||
strLstAddIfMissing(groupList, item->group); \
|
strLstAddIfMissing(groupList, item.group); \
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper to warn when an owner is missing and must be remapped
|
// Helper to warn when an owner is missing and must be remapped
|
||||||
@ -719,9 +719,9 @@ restoreManifestOwner(const Manifest *const manifest, const String **const rootRe
|
|||||||
bool groupNull = false;
|
bool groupNull = false;
|
||||||
StringList *groupList = strLstNew();
|
StringList *groupList = strLstNew();
|
||||||
|
|
||||||
RESTORE_MANIFEST_OWNER_GET(File);
|
RESTORE_MANIFEST_OWNER_GET(File, );
|
||||||
RESTORE_MANIFEST_OWNER_GET(Link);
|
RESTORE_MANIFEST_OWNER_GET(Link, *(ManifestLink *));
|
||||||
RESTORE_MANIFEST_OWNER_GET(Path);
|
RESTORE_MANIFEST_OWNER_GET(Path, *(ManifestPath *));
|
||||||
|
|
||||||
// Update users and groups in the manifest (this can only be done as root)
|
// Update users and groups in the manifest (this can only be done as root)
|
||||||
// -------------------------------------------------------------------------------------------------------------------------
|
// -------------------------------------------------------------------------------------------------------------------------
|
||||||
@ -918,14 +918,15 @@ restoreCleanInfoListCallback(void *data, const StorageInfo *info)
|
|||||||
{
|
{
|
||||||
case storageTypeFile:
|
case storageTypeFile:
|
||||||
{
|
{
|
||||||
const ManifestFile *manifestFile = manifestFileFindDefault(cleanData->manifest, manifestName, NULL);
|
if (manifestFileExists(cleanData->manifest, manifestName) &&
|
||||||
|
manifestLinkFindDefault(cleanData->manifest, manifestName, NULL) == NULL)
|
||||||
if (manifestFile != NULL && manifestLinkFindDefault(cleanData->manifest, manifestName, NULL) == NULL)
|
|
||||||
{
|
{
|
||||||
|
const ManifestFile manifestFile = manifestFileFind(cleanData->manifest, manifestName);
|
||||||
|
|
||||||
restoreCleanOwnership(
|
restoreCleanOwnership(
|
||||||
pgPath, manifestFile->user, cleanData->rootReplaceUser, manifestFile->group, cleanData->rootReplaceGroup,
|
pgPath, manifestFile.user, cleanData->rootReplaceUser, manifestFile.group, cleanData->rootReplaceGroup,
|
||||||
info->userId, info->groupId, false);
|
info->userId, info->groupId, false);
|
||||||
restoreCleanMode(pgPath, manifestFile->mode, info);
|
restoreCleanMode(pgPath, manifestFile.mode, info);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -1133,7 +1134,7 @@ restoreCleanBuild(const Manifest *const manifest, const String *const rootReplac
|
|||||||
|
|
||||||
// Skip the tablespace_map file when present so PostgreSQL does not rewrite links in pg_tblspc. The tablespace links will be
|
// Skip the tablespace_map file when present so PostgreSQL does not rewrite links in pg_tblspc. The tablespace links will be
|
||||||
// created after paths are cleaned.
|
// created after paths are cleaned.
|
||||||
if (manifestFileFindDefault(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_TABLESPACEMAP), NULL) != NULL &&
|
if (manifestFileExists(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_TABLESPACEMAP)) &&
|
||||||
manifestData(manifest)->pgVersion >= PG_VERSION_TABLESPACE_MAP)
|
manifestData(manifest)->pgVersion >= PG_VERSION_TABLESPACE_MAP)
|
||||||
{
|
{
|
||||||
LOG_DETAIL_FMT("skip '" PG_FILE_TABLESPACEMAP "' -- tablespace links will be created based on mappings");
|
LOG_DETAIL_FMT("skip '" PG_FILE_TABLESPACEMAP "' -- tablespace links will be created based on mappings");
|
||||||
@ -1142,7 +1143,7 @@ restoreCleanBuild(const Manifest *const manifest, const String *const rootReplac
|
|||||||
|
|
||||||
// Skip postgresql.auto.conf if preserve is set and the PostgreSQL version supports recovery GUCs
|
// Skip postgresql.auto.conf if preserve is set and the PostgreSQL version supports recovery GUCs
|
||||||
if (manifestData(manifest)->pgVersion >= PG_VERSION_RECOVERY_GUC && cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_PRESERVE &&
|
if (manifestData(manifest)->pgVersion >= PG_VERSION_RECOVERY_GUC && cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_PRESERVE &&
|
||||||
manifestFileFindDefault(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF), NULL) != NULL)
|
manifestFileExists(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)))
|
||||||
{
|
{
|
||||||
LOG_DETAIL_FMT("skip '" PG_FILE_POSTGRESQLAUTOCONF "' -- recovery type is preserve");
|
LOG_DETAIL_FMT("skip '" PG_FILE_POSTGRESQLAUTOCONF "' -- recovery type is preserve");
|
||||||
manifestFileRemove(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF));
|
manifestFileRemove(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF));
|
||||||
@ -1341,11 +1342,11 @@ restoreSelectiveExpression(Manifest *manifest)
|
|||||||
|
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFile(manifest, fileIdx);
|
const String *const fileName = manifestFileNameGet(manifest, fileIdx);
|
||||||
|
|
||||||
if (regExpMatch(baseRegExp, file->name) || regExpMatch(tablespaceRegExp, file->name))
|
if (regExpMatch(baseRegExp, fileName) || regExpMatch(tablespaceRegExp, fileName))
|
||||||
{
|
{
|
||||||
String *dbId = strBase(strPath(file->name));
|
String *dbId = strBase(strPath(fileName));
|
||||||
|
|
||||||
// In the highly unlikely event that a system database was somehow added after the backup began, it will only be
|
// In the highly unlikely event that a system database was somehow added after the backup began, it will only be
|
||||||
// found in the file list and not the manifest db section, so add it to the system database list
|
// found in the file list and not the manifest db section, so add it to the system database list
|
||||||
@ -1907,14 +1908,18 @@ restoreProcessQueueComparator(const void *item1, const void *item2)
|
|||||||
ASSERT(item1 != NULL);
|
ASSERT(item1 != NULL);
|
||||||
ASSERT(item2 != NULL);
|
ASSERT(item2 != NULL);
|
||||||
|
|
||||||
|
// Unpack files
|
||||||
|
ManifestFile file1 = manifestFileUnpack(*(const ManifestFilePack **)item1);
|
||||||
|
ManifestFile file2 = manifestFileUnpack(*(const ManifestFilePack **)item2);
|
||||||
|
|
||||||
// If the size differs then that's enough to determine order
|
// If the size differs then that's enough to determine order
|
||||||
if ((*(ManifestFile **)item1)->size < (*(ManifestFile **)item2)->size)
|
if (file1.size < file2.size)
|
||||||
FUNCTION_TEST_RETURN(-1);
|
FUNCTION_TEST_RETURN(-1);
|
||||||
else if ((*(ManifestFile **)item1)->size > (*(ManifestFile **)item2)->size)
|
else if (file1.size > file2.size)
|
||||||
FUNCTION_TEST_RETURN(1);
|
FUNCTION_TEST_RETURN(1);
|
||||||
|
|
||||||
// If size is the same then use name to generate a deterministic ordering (names must be unique)
|
// If size is the same then use name to generate a deterministic ordering (names must be unique)
|
||||||
FUNCTION_TEST_RETURN(strCmp((*(ManifestFile **)item1)->name, (*(ManifestFile **)item2)->name));
|
FUNCTION_TEST_RETURN(strCmp(file1.name, file2.name));
|
||||||
}
|
}
|
||||||
|
|
||||||
static uint64_t
|
static uint64_t
|
||||||
@ -1960,7 +1965,8 @@ restoreProcessQueue(Manifest *manifest, List **queueList)
|
|||||||
// Now put all files into the processing queues
|
// Now put all files into the processing queues
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFile(manifest, fileIdx);
|
const ManifestFilePack *const filePack = manifestFilePackGet(manifest, fileIdx);
|
||||||
|
const ManifestFile file = manifestFileUnpack(filePack);
|
||||||
|
|
||||||
// Find the target that contains this file
|
// Find the target that contains this file
|
||||||
unsigned int targetIdx = 0;
|
unsigned int targetIdx = 0;
|
||||||
@ -1970,7 +1976,7 @@ restoreProcessQueue(Manifest *manifest, List **queueList)
|
|||||||
// A target should always be found
|
// A target should always be found
|
||||||
CHECK(FormatError, targetIdx < strLstSize(targetList), "backup target not found");
|
CHECK(FormatError, targetIdx < strLstSize(targetList), "backup target not found");
|
||||||
|
|
||||||
if (strBeginsWith(file->name, strLstGet(targetList, targetIdx)))
|
if (strBeginsWith(file.name, strLstGet(targetList, targetIdx)))
|
||||||
break;
|
break;
|
||||||
|
|
||||||
targetIdx++;
|
targetIdx++;
|
||||||
@ -1978,10 +1984,10 @@ restoreProcessQueue(Manifest *manifest, List **queueList)
|
|||||||
while (1);
|
while (1);
|
||||||
|
|
||||||
// Add file to queue
|
// Add file to queue
|
||||||
lstAdd(*(List **)lstGet(*queueList, targetIdx), &file);
|
lstAdd(*(List **)lstGet(*queueList, targetIdx), &filePack);
|
||||||
|
|
||||||
// Add size to total
|
// Add size to total
|
||||||
result += file->size;
|
result += file.size;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sort the queues
|
// Sort the queues
|
||||||
@ -2052,8 +2058,8 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
|
|||||||
{
|
{
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
|
const ManifestFile file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
|
||||||
bool zeroed = restoreFileZeroed(file->name, zeroExp);
|
bool zeroed = restoreFileZeroed(file.name, zeroExp);
|
||||||
bool copy = pckReadBoolP(protocolParallelJobResult(job));
|
bool copy = pckReadBoolP(protocolParallelJobResult(job));
|
||||||
|
|
||||||
String *log = strCatZ(strNew(), "restore");
|
String *log = strCatZ(strNew(), "restore");
|
||||||
@ -2063,7 +2069,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
|
|||||||
strCatZ(log, " zeroed");
|
strCatZ(log, " zeroed");
|
||||||
|
|
||||||
// Add filename
|
// Add filename
|
||||||
strCatFmt(log, " file %s", strZ(restoreFilePgPath(manifest, file->name)));
|
strCatFmt(log, " file %s", strZ(restoreFilePgPath(manifest, file.name)));
|
||||||
|
|
||||||
// If not copied and not zeroed add details to explain why it was not copied
|
// If not copied and not zeroed add details to explain why it was not copied
|
||||||
if (!copy && !zeroed)
|
if (!copy && !zeroed)
|
||||||
@ -2074,8 +2080,8 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
|
|||||||
if (cfgOptionBool(cfgOptForce))
|
if (cfgOptionBool(cfgOptForce))
|
||||||
{
|
{
|
||||||
strCatFmt(
|
strCatFmt(
|
||||||
log, "exists and matches size %" PRIu64 " and modification time %" PRIu64, file->size,
|
log, "exists and matches size %" PRIu64 " and modification time %" PRIu64, file.size,
|
||||||
(uint64_t)file->timestamp);
|
(uint64_t)file.timestamp);
|
||||||
}
|
}
|
||||||
// Else a checksum delta or file is zero-length
|
// Else a checksum delta or file is zero-length
|
||||||
else
|
else
|
||||||
@ -2083,7 +2089,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
|
|||||||
strCatZ(log, "exists and ");
|
strCatZ(log, "exists and ");
|
||||||
|
|
||||||
// No need to copy zero-length files
|
// No need to copy zero-length files
|
||||||
if (file->size == 0)
|
if (file.size == 0)
|
||||||
{
|
{
|
||||||
strCatZ(log, "is zero size");
|
strCatZ(log, "is zero size");
|
||||||
}
|
}
|
||||||
@ -2094,12 +2100,12 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Add size and percent complete
|
// Add size and percent complete
|
||||||
sizeRestored += file->size;
|
sizeRestored += file.size;
|
||||||
strCatFmt(log, " (%s, %" PRIu64 "%%)", strZ(strSizeFormat(file->size)), sizeRestored * 100 / sizeTotal);
|
strCatFmt(log, " (%s, %" PRIu64 "%%)", strZ(strSizeFormat(file.size)), sizeRestored * 100 / sizeTotal);
|
||||||
|
|
||||||
// If not zero-length add the checksum
|
// If not zero-length add the checksum
|
||||||
if (file->size != 0 && !zeroed)
|
if (file.size != 0 && !zeroed)
|
||||||
strCatFmt(log, " checksum %s", file->checksumSha1);
|
strCatFmt(log, " checksum %s", file.checksumSha1);
|
||||||
|
|
||||||
LOG_DETAIL_PID(protocolParallelJobProcessId(job), strZ(log));
|
LOG_DETAIL_PID(protocolParallelJobProcessId(job), strZ(log));
|
||||||
}
|
}
|
||||||
@ -2178,24 +2184,24 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
|
|||||||
|
|
||||||
if (!lstEmpty(queue))
|
if (!lstEmpty(queue))
|
||||||
{
|
{
|
||||||
const ManifestFile *file = *(ManifestFile **)lstGet(queue, 0);
|
const ManifestFile file = manifestFileUnpack(*(ManifestFilePack **)lstGet(queue, 0));
|
||||||
|
|
||||||
// Create restore job
|
// Create restore job
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
|
||||||
PackWrite *const param = protocolCommandParam(command);
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
|
|
||||||
pckWriteStrP(param, file->name);
|
pckWriteStrP(param, file.name);
|
||||||
pckWriteU32P(param, jobData->repoIdx);
|
pckWriteU32P(param, jobData->repoIdx);
|
||||||
pckWriteStrP(param, file->reference != NULL ? file->reference : manifestData(jobData->manifest)->backupLabel);
|
pckWriteStrP(param, file.reference != NULL ? file.reference : manifestData(jobData->manifest)->backupLabel);
|
||||||
pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType);
|
pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType);
|
||||||
pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file->name));
|
pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file.name));
|
||||||
pckWriteStrP(param, STR(file->checksumSha1));
|
pckWriteStrP(param, STR(file.checksumSha1));
|
||||||
pckWriteBoolP(param, restoreFileZeroed(file->name, jobData->zeroExp));
|
pckWriteBoolP(param, restoreFileZeroed(file.name, jobData->zeroExp));
|
||||||
pckWriteU64P(param, file->size);
|
pckWriteU64P(param, file.size);
|
||||||
pckWriteTimeP(param, file->timestamp);
|
pckWriteTimeP(param, file.timestamp);
|
||||||
pckWriteModeP(param, file->mode);
|
pckWriteModeP(param, file.mode);
|
||||||
pckWriteStrP(param, restoreManifestOwnerReplace(file->user, jobData->rootReplaceUser));
|
pckWriteStrP(param, restoreManifestOwnerReplace(file.user, jobData->rootReplaceUser));
|
||||||
pckWriteStrP(param, restoreManifestOwnerReplace(file->group, jobData->rootReplaceGroup));
|
pckWriteStrP(param, restoreManifestOwnerReplace(file.group, jobData->rootReplaceGroup));
|
||||||
pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart);
|
pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart);
|
||||||
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta));
|
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta));
|
||||||
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce));
|
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce));
|
||||||
@ -2207,7 +2213,7 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
|
|||||||
// Assign job to result
|
// Assign job to result
|
||||||
MEM_CONTEXT_PRIOR_BEGIN()
|
MEM_CONTEXT_PRIOR_BEGIN()
|
||||||
{
|
{
|
||||||
result = protocolParallelJobNew(VARSTR(file->name), command);
|
result = protocolParallelJobNew(VARSTR(file.name), command);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_PRIOR_END();
|
MEM_CONTEXT_PRIOR_END();
|
||||||
|
|
||||||
|
@ -902,7 +902,7 @@ verifyBackup(void *data)
|
|||||||
{
|
{
|
||||||
do
|
do
|
||||||
{
|
{
|
||||||
const ManifestFile *fileData = manifestFile(jobData->manifest, jobData->manifestFileIdx);
|
const ManifestFile fileData = manifestFile(jobData->manifest, jobData->manifestFileIdx);
|
||||||
|
|
||||||
String *filePathName = NULL;
|
String *filePathName = NULL;
|
||||||
|
|
||||||
@ -910,16 +910,16 @@ verifyBackup(void *data)
|
|||||||
backupResult->totalFileVerify++;
|
backupResult->totalFileVerify++;
|
||||||
|
|
||||||
// Check if the file is referenced in a prior backup
|
// Check if the file is referenced in a prior backup
|
||||||
if (fileData->reference != NULL)
|
if (fileData.reference != NULL)
|
||||||
{
|
{
|
||||||
// If the prior backup is not in the result list, then that backup was never processed (likely due to the --set
|
// If the prior backup is not in the result list, then that backup was never processed (likely due to the --set
|
||||||
// option) so verify the file
|
// option) so verify the file
|
||||||
unsigned int backupPriorIdx = lstFindIdx(jobData->backupResultList, &fileData->reference);
|
unsigned int backupPriorIdx = lstFindIdx(jobData->backupResultList, &fileData.reference);
|
||||||
|
|
||||||
if (backupPriorIdx == LIST_NOT_FOUND)
|
if (backupPriorIdx == LIST_NOT_FOUND)
|
||||||
{
|
{
|
||||||
filePathName = strNewFmt(
|
filePathName = strNewFmt(
|
||||||
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name),
|
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData.reference), strZ(fileData.name),
|
||||||
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
|
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
|
||||||
}
|
}
|
||||||
// Else the backup this file references has a result so check the processing state for the referenced backup
|
// Else the backup this file references has a result so check the processing state for the referenced backup
|
||||||
@ -931,14 +931,14 @@ verifyBackup(void *data)
|
|||||||
if (!backupResultPrior->fileVerifyComplete)
|
if (!backupResultPrior->fileVerifyComplete)
|
||||||
{
|
{
|
||||||
filePathName = strNewFmt(
|
filePathName = strNewFmt(
|
||||||
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name),
|
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData.reference), strZ(fileData.name),
|
||||||
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
|
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
|
||||||
}
|
}
|
||||||
// Else skip verification
|
// Else skip verification
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
String *priorFile = strNewFmt(
|
String *priorFile = strNewFmt(
|
||||||
"%s/%s%s", strZ(fileData->reference), strZ(fileData->name),
|
"%s/%s%s", strZ(fileData.reference), strZ(fileData.name),
|
||||||
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
|
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
|
||||||
|
|
||||||
unsigned int backupPriorInvalidIdx = lstFindIdx(backupResultPrior->invalidFileList, &priorFile);
|
unsigned int backupPriorInvalidIdx = lstFindIdx(backupResultPrior->invalidFileList, &priorFile);
|
||||||
@ -965,7 +965,7 @@ verifyBackup(void *data)
|
|||||||
else
|
else
|
||||||
{
|
{
|
||||||
filePathName = strNewFmt(
|
filePathName = strNewFmt(
|
||||||
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(backupResult->backupLabel), strZ(fileData->name),
|
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(backupResult->backupLabel), strZ(fileData.name),
|
||||||
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
|
strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -978,8 +978,8 @@ verifyBackup(void *data)
|
|||||||
|
|
||||||
pckWriteStrP(param, filePathName);
|
pckWriteStrP(param, filePathName);
|
||||||
// If the checksum is not present in the manifest, it will be calculated by manifest load
|
// If the checksum is not present in the manifest, it will be calculated by manifest load
|
||||||
pckWriteStrP(param, STR(fileData->checksumSha1));
|
pckWriteStrP(param, STR(fileData.checksumSha1));
|
||||||
pckWriteU64P(param, fileData->size);
|
pckWriteU64P(param, fileData.size);
|
||||||
pckWriteStrP(param, jobData->backupCipherPass);
|
pckWriteStrP(param, jobData->backupCipherPass);
|
||||||
|
|
||||||
// Assign job to result (prepend backup label being processed to the key since some files are in a prior backup)
|
// Assign job to result (prepend backup label being processed to the key since some files are in a prior backup)
|
||||||
|
@ -482,3 +482,80 @@ cvtZToUInt64(const char *value)
|
|||||||
|
|
||||||
FUNCTION_TEST_RETURN(cvtZToUInt64Base(value, 10));
|
FUNCTION_TEST_RETURN(cvtZToUInt64Base(value, 10));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**********************************************************************************************************************************/
|
||||||
|
void
|
||||||
|
cvtUInt64ToVarInt128(uint64_t value, uint8_t *const buffer, size_t *const bufferPos, const size_t bufferSize)
|
||||||
|
{
|
||||||
|
FUNCTION_TEST_BEGIN();
|
||||||
|
FUNCTION_TEST_PARAM(UINT64, value);
|
||||||
|
FUNCTION_TEST_PARAM_P(VOID, buffer);
|
||||||
|
FUNCTION_TEST_PARAM(UINT64, bufferSize);
|
||||||
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
|
ASSERT(buffer != NULL);
|
||||||
|
ASSERT(bufferPos != NULL);
|
||||||
|
ASSERT(bufferSize > *bufferPos);
|
||||||
|
|
||||||
|
// Keep encoding bytes while the remaining value is greater than 7 bits
|
||||||
|
while (value >= 0x80)
|
||||||
|
{
|
||||||
|
// Encode the lower order 7 bits, adding the continuation bit to indicate there is more data
|
||||||
|
buffer[*bufferPos] = (unsigned char)value | 0x80;
|
||||||
|
|
||||||
|
// Shift the value to remove bits that have been encoded
|
||||||
|
value >>= 7;
|
||||||
|
|
||||||
|
// Keep track of size so we know how many bytes to write out
|
||||||
|
(*bufferPos)++;
|
||||||
|
|
||||||
|
// Make sure the buffer won't overflow
|
||||||
|
if (*bufferPos >= bufferSize)
|
||||||
|
THROW(AssertError, "buffer overflow");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Encode the last 7 bits of value
|
||||||
|
buffer[*bufferPos] = (unsigned char)value;
|
||||||
|
(*bufferPos)++;
|
||||||
|
|
||||||
|
FUNCTION_TEST_RETURN_VOID();
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t
|
||||||
|
cvtUInt64FromVarInt128(const uint8_t *const value, size_t *const valuePos)
|
||||||
|
{
|
||||||
|
FUNCTION_TEST_BEGIN();
|
||||||
|
FUNCTION_TEST_PARAM_P(VOID, value);
|
||||||
|
FUNCTION_TEST_PARAM_P(UINT64, valuePos);
|
||||||
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
|
ASSERT(value != NULL);
|
||||||
|
ASSERT(valuePos != NULL);
|
||||||
|
|
||||||
|
// Decode all bytes
|
||||||
|
uint64_t result = 0;
|
||||||
|
uint8_t byte;
|
||||||
|
|
||||||
|
for (unsigned int valueIdx = 0; valueIdx < CVT_VARINT128_BUFFER_SIZE; valueIdx++)
|
||||||
|
{
|
||||||
|
// Get the next encoded byte
|
||||||
|
byte = value[*valuePos];
|
||||||
|
|
||||||
|
// Shift the lower order 7 encoded bits into the uint64 in reverse order
|
||||||
|
result |= (uint64_t)(byte & 0x7f) << (7 * valueIdx);
|
||||||
|
|
||||||
|
// Increment value position to indicate that the byte has been processed
|
||||||
|
(*valuePos)++;
|
||||||
|
|
||||||
|
// Done if the high order bit is not set to indicate more data
|
||||||
|
if (byte < 0x80)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// By this point all bytes should have been read so error if this is not the case. This could be due to a coding error or
|
||||||
|
// corrupton in the data stream.
|
||||||
|
if (byte >= 0x80)
|
||||||
|
THROW(FormatError, "unterminated varint-128 integer");
|
||||||
|
|
||||||
|
FUNCTION_TEST_RETURN(result);
|
||||||
|
}
|
||||||
|
@ -15,6 +15,7 @@ Required buffer sizes
|
|||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
#define CVT_BOOL_BUFFER_SIZE 6
|
#define CVT_BOOL_BUFFER_SIZE 6
|
||||||
#define CVT_BASE10_BUFFER_SIZE 64
|
#define CVT_BASE10_BUFFER_SIZE 64
|
||||||
|
#define CVT_VARINT128_BUFFER_SIZE 10
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
@ -85,6 +86,10 @@ size_t cvtUInt64ToZ(uint64_t value, char *buffer, size_t bufferSize);
|
|||||||
uint64_t cvtZToUInt64(const char *value);
|
uint64_t cvtZToUInt64(const char *value);
|
||||||
uint64_t cvtZToUInt64Base(const char *value, int base);
|
uint64_t cvtZToUInt64Base(const char *value, int base);
|
||||||
|
|
||||||
|
// Convert uint64 to base-128 varint and vice versa
|
||||||
|
void cvtUInt64ToVarInt128(uint64_t value, uint8_t *buffer, size_t *bufferPos, size_t bufferSize);
|
||||||
|
uint64_t cvtUInt64FromVarInt128(const uint8_t *value, size_t *valuePos);
|
||||||
|
|
||||||
// Convert boolean to zero-terminated string. Use cvtBoolToConstZ() whenever possible since it is more efficient.
|
// Convert boolean to zero-terminated string. Use cvtBoolToConstZ() whenever possible since it is more efficient.
|
||||||
size_t cvtBoolToZ(bool value, char *buffer, size_t bufferSize);
|
size_t cvtBoolToZ(bool value, char *buffer, size_t bufferSize);
|
||||||
const char *cvtBoolToConstZ(bool value);
|
const char *cvtBoolToConstZ(bool value);
|
||||||
|
@ -104,11 +104,6 @@ Array and object types:
|
|||||||
#include "common/type/convert.h"
|
#include "common/type/convert.h"
|
||||||
#include "common/type/pack.h"
|
#include "common/type/pack.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
|
||||||
Constants
|
|
||||||
***********************************************************************************************************************************/
|
|
||||||
#define PACK_UINT64_SIZE_MAX 10
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Map PackType types to the types that will be written into the pack. This hides the details of the type IDs from the user and allows
|
Map PackType types to the types that will be written into the pack. This hides the details of the type IDs from the user and allows
|
||||||
the IDs used in the pack to differ from the IDs the user sees.
|
the IDs used in the pack to differ from the IDs the user sees.
|
||||||
@ -495,7 +490,7 @@ pckReadU64Internal(PackRead *this)
|
|||||||
uint8_t byte;
|
uint8_t byte;
|
||||||
|
|
||||||
// Convert bytes from varint-128 encoding to a uint64
|
// Convert bytes from varint-128 encoding to a uint64
|
||||||
for (unsigned int bufferIdx = 0; bufferIdx < PACK_UINT64_SIZE_MAX; bufferIdx++)
|
for (unsigned int bufferIdx = 0; bufferIdx < CVT_VARINT128_BUFFER_SIZE; bufferIdx++)
|
||||||
{
|
{
|
||||||
// Get the next encoded byte
|
// Get the next encoded byte
|
||||||
pckReadBuffer(this, 1);
|
pckReadBuffer(this, 1);
|
||||||
@ -1409,7 +1404,7 @@ pckWriteBuffer(PackWrite *this, const Buffer *buffer)
|
|||||||
Pack an unsigned 64-bit integer to base-128 varint encoding
|
Pack an unsigned 64-bit integer to base-128 varint encoding
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
static void
|
static void
|
||||||
pckWriteU64Internal(PackWrite *this, uint64_t value)
|
pckWriteU64Internal(PackWrite *const this, const uint64_t value)
|
||||||
{
|
{
|
||||||
FUNCTION_TEST_BEGIN();
|
FUNCTION_TEST_BEGIN();
|
||||||
FUNCTION_TEST_PARAM(PACK_WRITE, this);
|
FUNCTION_TEST_PARAM(PACK_WRITE, this);
|
||||||
@ -1418,27 +1413,13 @@ pckWriteU64Internal(PackWrite *this, uint64_t value)
|
|||||||
|
|
||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
|
|
||||||
unsigned char buffer[PACK_UINT64_SIZE_MAX];
|
unsigned char buffer[CVT_VARINT128_BUFFER_SIZE];
|
||||||
size_t size = 0;
|
size_t size = 0;
|
||||||
|
|
||||||
// Convert uint64 to varint-128 encoding. Keep writing out bytes while the remaining value is greater than 7 bits.
|
cvtUInt64ToVarInt128(value, buffer, &size, sizeof(buffer));
|
||||||
while (value >= 0x80)
|
|
||||||
{
|
|
||||||
// Encode the lower order 7 bits, adding the continuation bit to indicate there is more data
|
|
||||||
buffer[size] = (unsigned char)value | 0x80;
|
|
||||||
|
|
||||||
// Shift the value to remove bits that have been encoded
|
|
||||||
value >>= 7;
|
|
||||||
|
|
||||||
// Keep track of size so we know how many bytes to write out
|
|
||||||
size++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Encode the last 7 bits of value
|
|
||||||
buffer[size] = (unsigned char)value;
|
|
||||||
|
|
||||||
// Write encoded bytes to the buffer
|
// Write encoded bytes to the buffer
|
||||||
pckWriteBuffer(this, BUF(buffer, size + 1));
|
pckWriteBuffer(this, BUF(buffer, size));
|
||||||
|
|
||||||
FUNCTION_TEST_RETURN_VOID();
|
FUNCTION_TEST_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
@ -370,22 +370,22 @@ infoBackupDataAdd(const InfoBackup *this, const Manifest *manifest)
|
|||||||
|
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFile(manifest, fileIdx);
|
const ManifestFile file = manifestFile(manifest, fileIdx);
|
||||||
|
|
||||||
backupSize += file->size;
|
backupSize += file.size;
|
||||||
backupRepoSize += file->sizeRepo > 0 ? file->sizeRepo : file->size;
|
backupRepoSize += file.sizeRepo > 0 ? file.sizeRepo : file.size;
|
||||||
|
|
||||||
// If a reference to a file exists, then it is in a previous backup and the delta calculation was already done
|
// If a reference to a file exists, then it is in a previous backup and the delta calculation was already done
|
||||||
if (file->reference != NULL)
|
if (file.reference != NULL)
|
||||||
strLstAddIfMissing(referenceList, file->reference);
|
strLstAddIfMissing(referenceList, file.reference);
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
backupSizeDelta += file->size;
|
backupSizeDelta += file.size;
|
||||||
backupRepoSizeDelta += file->sizeRepo > 0 ? file->sizeRepo : file->size;
|
backupRepoSizeDelta += file.sizeRepo > 0 ? file.sizeRepo : file.size;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Is there an error in the file?
|
// Is there an error in the file?
|
||||||
if (file->checksumPageError)
|
if (file.checksumPageError)
|
||||||
backupError = true;
|
backupError = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -198,59 +198,208 @@ manifestDbAdd(Manifest *this, const ManifestDb *db)
|
|||||||
FUNCTION_TEST_RETURN_VOID();
|
FUNCTION_TEST_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Base time used as a delta to reduce the size of packed timestamps. This will be set on the first call to manifestFilePack().
|
||||||
|
static time_t manifestPackBaseTime = -1;
|
||||||
|
|
||||||
|
// Flags used to reduce the size of packed data. They should be ordered from most to least likely and can be reordered at will.
|
||||||
|
typedef enum
|
||||||
|
{
|
||||||
|
manifestFilePackFlagReference,
|
||||||
|
manifestFilePackFlagChecksumPage,
|
||||||
|
manifestFilePackFlagChecksumPageError,
|
||||||
|
manifestFilePackFlagChecksumPageErrorList,
|
||||||
|
} ManifestFilePackFlag;
|
||||||
|
|
||||||
|
// Pack file into a compact format to save memory
|
||||||
|
static ManifestFilePack *
|
||||||
|
manifestFilePack(const ManifestFile *const file)
|
||||||
|
{
|
||||||
|
FUNCTION_TEST_BEGIN();
|
||||||
|
FUNCTION_TEST_PARAM(MANIFEST_FILE, file);
|
||||||
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
|
uint8_t buffer[512];
|
||||||
|
size_t bufferPos = 0;
|
||||||
|
|
||||||
|
// Flags
|
||||||
|
uint64_t flag = 0;
|
||||||
|
|
||||||
|
if (file->checksumPage)
|
||||||
|
flag |= 1 << manifestFilePackFlagChecksumPage;
|
||||||
|
|
||||||
|
if (file->checksumPageError)
|
||||||
|
flag |= 1 << manifestFilePackFlagChecksumPageError;
|
||||||
|
|
||||||
|
if (file->checksumPageErrorList != NULL)
|
||||||
|
flag |= 1 << manifestFilePackFlagChecksumPageErrorList;
|
||||||
|
|
||||||
|
if (file->reference != NULL)
|
||||||
|
flag |= 1 << manifestFilePackFlagReference;
|
||||||
|
|
||||||
|
cvtUInt64ToVarInt128(flag, buffer, &bufferPos, sizeof(buffer));
|
||||||
|
|
||||||
|
// Size
|
||||||
|
cvtUInt64ToVarInt128(file->size, buffer, &bufferPos, sizeof(buffer));
|
||||||
|
|
||||||
|
// Use the first timestamp that appears as the base for all other timestamps. Ideally we would like a timestamp as close to the
|
||||||
|
// middle as possible but it doesn't seem worth doing the calculation.
|
||||||
|
if (manifestPackBaseTime == -1)
|
||||||
|
manifestPackBaseTime = file->timestamp;
|
||||||
|
|
||||||
|
// Timestamp
|
||||||
|
cvtUInt64ToVarInt128(cvtInt64ToZigZag(manifestPackBaseTime - file->timestamp), buffer, &bufferPos, sizeof(buffer));
|
||||||
|
|
||||||
|
// SHA1 checksum
|
||||||
|
strcpy((char *)buffer + bufferPos, file->checksumSha1);
|
||||||
|
bufferPos += HASH_TYPE_SHA1_SIZE_HEX + 1;
|
||||||
|
|
||||||
|
// Reference
|
||||||
|
if (file->reference != NULL)
|
||||||
|
cvtUInt64ToVarInt128((uintptr_t)file->reference, buffer, &bufferPos, sizeof(buffer));
|
||||||
|
|
||||||
|
// Mode
|
||||||
|
cvtUInt64ToVarInt128(file->mode, buffer, &bufferPos, sizeof(buffer));
|
||||||
|
|
||||||
|
// User/group
|
||||||
|
cvtUInt64ToVarInt128((uintptr_t)file->user, buffer, &bufferPos, sizeof(buffer));
|
||||||
|
cvtUInt64ToVarInt128((uintptr_t)file->group, buffer, &bufferPos, sizeof(buffer));
|
||||||
|
|
||||||
|
// Repo size
|
||||||
|
cvtUInt64ToVarInt128(file->sizeRepo, buffer, &bufferPos, sizeof(buffer));
|
||||||
|
|
||||||
|
// Allocate memory for the file pack
|
||||||
|
uint8_t *const result = memNew(
|
||||||
|
sizeof(StringPub) + strSize(file->name) + 1 + bufferPos + (file->checksumPageErrorList != NULL ?
|
||||||
|
sizeof(StringPub) + strSize(file->checksumPageErrorList) + 1 : 0));
|
||||||
|
|
||||||
|
// Create string object for the file name
|
||||||
|
*(StringPub *)result = (StringPub){.size = (unsigned int)strSize(file->name), .buffer = (char *)result + sizeof(StringPub)};
|
||||||
|
size_t resultPos = sizeof(StringPub);
|
||||||
|
|
||||||
|
memcpy(result + resultPos, (uint8_t *)strZ(file->name), strSize(file->name) + 1);
|
||||||
|
resultPos += strSize(file->name) + 1;
|
||||||
|
|
||||||
|
// Copy pack data
|
||||||
|
memcpy(result + resultPos, buffer, bufferPos);
|
||||||
|
|
||||||
|
// Create string object for the checksum error list
|
||||||
|
if (file->checksumPageErrorList != NULL)
|
||||||
|
{
|
||||||
|
resultPos += bufferPos;
|
||||||
|
|
||||||
|
*(StringPub *)(result + resultPos) = (StringPub)
|
||||||
|
{.size = (unsigned int)strSize(file->checksumPageErrorList), .buffer = (char *)result + resultPos + sizeof(StringPub)};
|
||||||
|
resultPos += sizeof(StringPub);
|
||||||
|
|
||||||
|
memcpy(result + resultPos, (uint8_t *)strZ(file->checksumPageErrorList), strSize(file->checksumPageErrorList) + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
FUNCTION_TEST_RETURN((ManifestFilePack *)result);
|
||||||
|
}
|
||||||
|
|
||||||
|
ManifestFile
|
||||||
|
manifestFileUnpack(const ManifestFilePack *const filePack)
|
||||||
|
{
|
||||||
|
FUNCTION_TEST_BEGIN();
|
||||||
|
FUNCTION_TEST_PARAM_P(VOID, filePack);
|
||||||
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
|
ASSERT(filePack != NULL);
|
||||||
|
ASSERT(manifestPackBaseTime != -1);
|
||||||
|
|
||||||
|
ManifestFile result = {0};
|
||||||
|
size_t bufferPos = 0;
|
||||||
|
|
||||||
|
// Name
|
||||||
|
result.name = (const String *)filePack;
|
||||||
|
bufferPos += sizeof(StringPub) + strSize(result.name) + 1;
|
||||||
|
|
||||||
|
// Flags
|
||||||
|
const uint64_t flag = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
|
||||||
|
|
||||||
|
// Size
|
||||||
|
result.size = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
|
||||||
|
|
||||||
|
// Timestamp
|
||||||
|
result.timestamp =
|
||||||
|
manifestPackBaseTime - (time_t)cvtInt64FromZigZag(cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos));
|
||||||
|
|
||||||
|
// Checksum page
|
||||||
|
result.checksumPage = flag & (1 << manifestFilePackFlagChecksumPage) ? true : false;
|
||||||
|
|
||||||
|
// SHA1 checksum
|
||||||
|
memcpy(result.checksumSha1, (const uint8_t *)filePack + bufferPos, HASH_TYPE_SHA1_SIZE_HEX + 1);
|
||||||
|
bufferPos += HASH_TYPE_SHA1_SIZE_HEX + 1;
|
||||||
|
|
||||||
|
// Reference
|
||||||
|
if (flag & (1 << manifestFilePackFlagReference))
|
||||||
|
result.reference = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
|
||||||
|
|
||||||
|
// Mode
|
||||||
|
result.mode = (mode_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
|
||||||
|
|
||||||
|
// User/group
|
||||||
|
result.user = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
|
||||||
|
result.group = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
|
||||||
|
|
||||||
|
// Repo size
|
||||||
|
result.sizeRepo = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos);
|
||||||
|
|
||||||
|
// Checksum page error
|
||||||
|
result.checksumPageError = flag & (1 << manifestFilePackFlagChecksumPageError) ? true : false;
|
||||||
|
|
||||||
|
if (flag & (1 << manifestFilePackFlagChecksumPageErrorList))
|
||||||
|
result.checksumPageErrorList = (const String *)((const uint8_t *)filePack + bufferPos);
|
||||||
|
|
||||||
|
FUNCTION_TEST_RETURN(result);
|
||||||
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
manifestFileAdd(Manifest *this, const ManifestFile *file)
|
manifestFileAdd(Manifest *this, ManifestFile file)
|
||||||
{
|
{
|
||||||
FUNCTION_TEST_BEGIN();
|
FUNCTION_TEST_BEGIN();
|
||||||
FUNCTION_TEST_PARAM(MANIFEST, this);
|
FUNCTION_TEST_PARAM(MANIFEST, this);
|
||||||
|
FUNCTION_TEST_PARAM(VOID, file);
|
||||||
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
|
ASSERT(this != NULL);
|
||||||
|
ASSERT(file.name != NULL);
|
||||||
|
|
||||||
|
file.user = manifestOwnerCache(this, file.user);
|
||||||
|
file.group = manifestOwnerCache(this, file.group);
|
||||||
|
|
||||||
|
if (file.reference != NULL)
|
||||||
|
file.reference = strLstAddIfMissing(this->referenceList, file.reference);
|
||||||
|
|
||||||
|
MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
|
||||||
|
{
|
||||||
|
const ManifestFilePack *const filePack = manifestFilePack(&file);
|
||||||
|
lstAdd(this->pub.fileList, &filePack);
|
||||||
|
}
|
||||||
|
MEM_CONTEXT_END();
|
||||||
|
|
||||||
|
FUNCTION_TEST_RETURN_VOID();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update file pack by creating a new one and then freeing the old one
|
||||||
|
static void
|
||||||
|
manifestFilePackUpdate(Manifest *const this, ManifestFilePack **const filePack, const ManifestFile *const file)
|
||||||
|
{
|
||||||
|
FUNCTION_TEST_BEGIN();
|
||||||
|
FUNCTION_TEST_PARAM(MANIFEST, this);
|
||||||
|
FUNCTION_TEST_PARAM_P(VOID, filePack);
|
||||||
FUNCTION_TEST_PARAM(MANIFEST_FILE, file);
|
FUNCTION_TEST_PARAM(MANIFEST_FILE, file);
|
||||||
FUNCTION_TEST_END();
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
|
ASSERT(filePack != NULL);
|
||||||
ASSERT(file != NULL);
|
ASSERT(file != NULL);
|
||||||
ASSERT(file->name != NULL);
|
|
||||||
|
|
||||||
MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
|
MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
|
||||||
{
|
{
|
||||||
ManifestFile fileAdd =
|
ManifestFilePack *const filePackOld = *filePack;
|
||||||
{
|
*filePack = manifestFilePack(file);
|
||||||
.checksumPage = file->checksumPage,
|
memFree(filePackOld);
|
||||||
.checksumPageError = file->checksumPageError,
|
|
||||||
.checksumPageErrorList = varLstDup(file->checksumPageErrorList),
|
|
||||||
.group = manifestOwnerCache(this, file->group),
|
|
||||||
.mode = file->mode,
|
|
||||||
.name = strDup(file->name),
|
|
||||||
.size = file->size,
|
|
||||||
.sizeRepo = file->sizeRepo,
|
|
||||||
.timestamp = file->timestamp,
|
|
||||||
.user = manifestOwnerCache(this, file->user),
|
|
||||||
};
|
|
||||||
|
|
||||||
memcpy(fileAdd.checksumSha1, file->checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
|
|
||||||
|
|
||||||
if (file->reference != NULL)
|
|
||||||
{
|
|
||||||
// Search for the reference in the list
|
|
||||||
for (unsigned int referenceIdx = 0; referenceIdx < strLstSize(this->referenceList); referenceIdx++)
|
|
||||||
{
|
|
||||||
const String *found = strLstGet(this->referenceList, referenceIdx);
|
|
||||||
|
|
||||||
if (strEq(file->reference, found))
|
|
||||||
{
|
|
||||||
fileAdd.reference = found;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If not found then add it
|
|
||||||
if (fileAdd.reference == NULL)
|
|
||||||
{
|
|
||||||
strLstAdd(this->referenceList, file->reference);
|
|
||||||
fileAdd.reference = strLstGet(this->referenceList, strLstSize(this->referenceList) - 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
lstAdd(this->pub.fileList, &fileAdd);
|
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_END();
|
MEM_CONTEXT_END();
|
||||||
|
|
||||||
@ -364,7 +513,7 @@ manifestNewInternal(void)
|
|||||||
{
|
{
|
||||||
.memContext = memContextCurrent(),
|
.memContext = memContextCurrent(),
|
||||||
.dbList = lstNewP(sizeof(ManifestDb), .comparator = lstComparatorStr),
|
.dbList = lstNewP(sizeof(ManifestDb), .comparator = lstComparatorStr),
|
||||||
.fileList = lstNewP(sizeof(ManifestFile), .comparator = lstComparatorStr),
|
.fileList = lstNewP(sizeof(ManifestFilePack *), .comparator = lstComparatorStr),
|
||||||
.linkList = lstNewP(sizeof(ManifestLink), .comparator = lstComparatorStr),
|
.linkList = lstNewP(sizeof(ManifestLink), .comparator = lstComparatorStr),
|
||||||
.pathList = lstNewP(sizeof(ManifestPath), .comparator = lstComparatorStr),
|
.pathList = lstNewP(sizeof(ManifestPath), .comparator = lstComparatorStr),
|
||||||
.targetList = lstNewP(sizeof(ManifestTarget), .comparator = lstComparatorStr),
|
.targetList = lstNewP(sizeof(ManifestTarget), .comparator = lstComparatorStr),
|
||||||
@ -843,7 +992,7 @@ manifestBuildCallback(void *data, const StorageInfo *info)
|
|||||||
!strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL);
|
!strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL);
|
||||||
}
|
}
|
||||||
|
|
||||||
manifestFileAdd(buildData.manifest, &file);
|
manifestFileAdd(buildData.manifest, file);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1163,12 +1312,12 @@ manifestNewBuild(
|
|||||||
while (fileIdx < manifestFileTotal(this))
|
while (fileIdx < manifestFileTotal(this))
|
||||||
{
|
{
|
||||||
// If this file looks like a relation. Note that this never matches on _init forks.
|
// If this file looks like a relation. Note that this never matches on _init forks.
|
||||||
const ManifestFile *file = manifestFile(this, fileIdx);
|
const String *const filePathName = manifestFileNameGet(this, fileIdx);
|
||||||
|
|
||||||
if (regExpMatch(relationExp, file->name))
|
if (regExpMatch(relationExp, filePathName))
|
||||||
{
|
{
|
||||||
// Get the filename (without path)
|
// Get the filename (without path)
|
||||||
const char *fileName = strBaseZ(file->name);
|
const char *fileName = strBaseZ(filePathName);
|
||||||
size_t fileNameSize = strlen(fileName);
|
size_t fileNameSize = strlen(fileName);
|
||||||
|
|
||||||
// Strip off the numeric part of the relation
|
// Strip off the numeric part of the relation
|
||||||
@ -1193,8 +1342,8 @@ manifestNewBuild(
|
|||||||
{
|
{
|
||||||
// Determine if the relation is unlogged
|
// Determine if the relation is unlogged
|
||||||
String *relationInit = strNewFmt(
|
String *relationInit = strNewFmt(
|
||||||
"%.*s%s_init", (int)(strSize(file->name) - fileNameSize), strZ(file->name), relationFileId);
|
"%.*s%s_init", (int)(strSize(filePathName) - fileNameSize), strZ(filePathName), relationFileId);
|
||||||
lastRelationFileIdUnlogged = manifestFileFindDefault(this, relationInit, NULL) != NULL;
|
lastRelationFileIdUnlogged = manifestFileExists(this, relationInit);
|
||||||
strFree(relationInit);
|
strFree(relationInit);
|
||||||
|
|
||||||
// Save the file id so we don't need to do the lookup next time if it doesn't change
|
// Save the file id so we don't need to do the lookup next time if it doesn't change
|
||||||
@ -1204,7 +1353,7 @@ manifestNewBuild(
|
|||||||
// If relation is unlogged then remove it
|
// If relation is unlogged then remove it
|
||||||
if (lastRelationFileIdUnlogged)
|
if (lastRelationFileIdUnlogged)
|
||||||
{
|
{
|
||||||
manifestFileRemove(this, file->name);
|
manifestFileRemove(this, filePathName);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1262,13 +1411,13 @@ manifestBuildValidate(Manifest *this, bool delta, time_t copyStart, CompressType
|
|||||||
{
|
{
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFile(this, fileIdx);
|
const ManifestFile file = manifestFile(this, fileIdx);
|
||||||
|
|
||||||
// Check for timestamp in the future
|
// Check for timestamp in the future
|
||||||
if (file->timestamp > copyStart)
|
if (file.timestamp > copyStart)
|
||||||
{
|
{
|
||||||
LOG_WARN_FMT(
|
LOG_WARN_FMT(
|
||||||
"file '%s' has timestamp in the future, enabling delta checksum", strZ(manifestPathPg(file->name)));
|
"file '%s' has timestamp in the future, enabling delta checksum", strZ(manifestPathPg(file.name)));
|
||||||
|
|
||||||
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
|
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
|
||||||
break;
|
break;
|
||||||
@ -1337,29 +1486,30 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
|
|||||||
{
|
{
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFile(this, fileIdx);
|
const ManifestFile file = manifestFile(this, fileIdx);
|
||||||
const ManifestFile *filePrior = manifestFileFindDefault(manifestPrior, file->name, NULL);
|
|
||||||
|
|
||||||
// If file was found in prior manifest then perform checks
|
// If file was found in prior manifest then perform checks
|
||||||
if (filePrior != NULL)
|
if (manifestFileExists(manifestPrior, file.name))
|
||||||
{
|
{
|
||||||
|
const ManifestFile filePrior = manifestFileFind(manifestPrior, file.name);
|
||||||
|
|
||||||
// Check for timestamp earlier than the prior backup
|
// Check for timestamp earlier than the prior backup
|
||||||
if (file->timestamp < filePrior->timestamp)
|
if (file.timestamp < filePrior.timestamp)
|
||||||
{
|
{
|
||||||
LOG_WARN_FMT(
|
LOG_WARN_FMT(
|
||||||
"file '%s' has timestamp earlier than prior backup, enabling delta checksum",
|
"file '%s' has timestamp earlier than prior backup, enabling delta checksum",
|
||||||
strZ(manifestPathPg(file->name)));
|
strZ(manifestPathPg(file.name)));
|
||||||
|
|
||||||
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
|
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for size change with no timestamp change
|
// Check for size change with no timestamp change
|
||||||
if (file->size != filePrior->size && file->timestamp == filePrior->timestamp)
|
if (file.size != filePrior.size && file.timestamp == filePrior.timestamp)
|
||||||
{
|
{
|
||||||
LOG_WARN_FMT(
|
LOG_WARN_FMT(
|
||||||
"file '%s' has same timestamp as prior but different size, enabling delta checksum",
|
"file '%s' has same timestamp as prior but different size, enabling delta checksum",
|
||||||
strZ(manifestPathPg(file->name)));
|
strZ(manifestPathPg(file.name)));
|
||||||
|
|
||||||
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
|
this->pub.data.backupOptionDelta = BOOL_TRUE_VAR;
|
||||||
break;
|
break;
|
||||||
@ -1375,17 +1525,20 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
|
|||||||
|
|
||||||
for (unsigned int fileIdx = 0; fileIdx < lstSize(this->pub.fileList); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < lstSize(this->pub.fileList); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFile(this, fileIdx);
|
const ManifestFile file = manifestFile(this, fileIdx);
|
||||||
const ManifestFile *filePrior = manifestFileFindDefault(manifestPrior, file->name, NULL);
|
|
||||||
|
|
||||||
// Check if prior file can be used
|
// Check if prior file can be used
|
||||||
if (filePrior != NULL && file->size == filePrior->size &&
|
if (manifestFileExists(manifestPrior, file.name))
|
||||||
(delta || file->size == 0 || file->timestamp == filePrior->timestamp))
|
{
|
||||||
|
const ManifestFile filePrior = manifestFileFind(manifestPrior, file.name);
|
||||||
|
|
||||||
|
if (file.size == filePrior.size && (delta || file.size == 0 || file.timestamp == filePrior.timestamp))
|
||||||
{
|
{
|
||||||
manifestFileUpdate(
|
manifestFileUpdate(
|
||||||
this, file->name, file->size, filePrior->sizeRepo, filePrior->checksumSha1,
|
this, file.name, file.size, filePrior.sizeRepo, filePrior.checksumSha1,
|
||||||
VARSTR(filePrior->reference != NULL ? filePrior->reference : manifestPrior->pub.data.backupLabel),
|
VARSTR(filePrior.reference != NULL ? filePrior.reference : manifestPrior->pub.data.backupLabel),
|
||||||
filePrior->checksumPage, filePrior->checksumPageError, filePrior->checksumPageErrorList);
|
filePrior.checksumPage, filePrior.checksumPageError, filePrior.checksumPageErrorList);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1618,7 +1771,7 @@ manifestLoadCallback(void *callbackData, const String *section, const String *ke
|
|||||||
const Variant *checksumPageErrorList = kvGetDefault(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, NULL);
|
const Variant *checksumPageErrorList = kvGetDefault(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, NULL);
|
||||||
|
|
||||||
if (checksumPageErrorList != NULL)
|
if (checksumPageErrorList != NULL)
|
||||||
file.checksumPageErrorList = varVarLst(checksumPageErrorList);
|
file.checksumPageErrorList = jsonFromVar(checksumPageErrorList);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (kvKeyExists(fileKv, MANIFEST_KEY_GROUP_VAR))
|
if (kvKeyExists(fileKv, MANIFEST_KEY_GROUP_VAR))
|
||||||
@ -1640,7 +1793,7 @@ manifestLoadCallback(void *callbackData, const String *section, const String *ke
|
|||||||
}
|
}
|
||||||
|
|
||||||
lstAdd(loadData->fileFoundList, &valueFound);
|
lstAdd(loadData->fileFoundList, &valueFound);
|
||||||
manifestFileAdd(manifest, &file);
|
manifestFileAdd(manifest, file);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_END();
|
MEM_CONTEXT_END();
|
||||||
}
|
}
|
||||||
@ -1926,17 +2079,20 @@ manifestNewLoad(IoRead *read)
|
|||||||
// Process file defaults
|
// Process file defaults
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
|
||||||
{
|
{
|
||||||
ManifestFile *file = lstGet(this->pub.fileList, fileIdx);
|
ManifestFilePack **const filePack = lstGet(this->pub.fileList, fileIdx);
|
||||||
|
ManifestFile file = manifestFileUnpack(*filePack);
|
||||||
ManifestLoadFound *found = lstGet(loadData.fileFoundList, fileIdx);
|
ManifestLoadFound *found = lstGet(loadData.fileFoundList, fileIdx);
|
||||||
|
|
||||||
if (!found->group)
|
if (!found->group)
|
||||||
file->group = manifestOwnerCache(this, manifestOwnerGet(loadData.fileGroupDefault));
|
file.group = manifestOwnerCache(this, manifestOwnerGet(loadData.fileGroupDefault));
|
||||||
|
|
||||||
if (!found->mode)
|
if (!found->mode)
|
||||||
file->mode = loadData.fileModeDefault;
|
file.mode = loadData.fileModeDefault;
|
||||||
|
|
||||||
if (!found->user)
|
if (!found->user)
|
||||||
file->user = manifestOwnerCache(this, manifestOwnerGet(loadData.fileUserDefault));
|
file.user = manifestOwnerCache(this, manifestOwnerGet(loadData.fileUserDefault));
|
||||||
|
|
||||||
|
manifestFilePackUpdate(this, filePack, &file);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process link defaults
|
// Process link defaults
|
||||||
@ -2243,41 +2399,41 @@ manifestSaveCallback(void *callbackData, const String *sectionNext, InfoSave *in
|
|||||||
{
|
{
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFile(manifest, fileIdx);
|
const ManifestFile file = manifestFile(manifest, fileIdx);
|
||||||
KeyValue *fileKv = kvNew();
|
KeyValue *fileKv = kvNew();
|
||||||
|
|
||||||
// Save if the file size is not zero and the checksum exists. The checksum might not exist if this is a partial
|
// Save if the file size is not zero and the checksum exists. The checksum might not exist if this is a partial
|
||||||
// save performed during a backup.
|
// save performed during a backup.
|
||||||
if (file->size != 0 && file->checksumSha1[0] != 0)
|
if (file.size != 0 && file.checksumSha1[0] != 0)
|
||||||
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_VAR, VARSTRZ(file->checksumSha1));
|
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_VAR, VARSTRZ(file.checksumSha1));
|
||||||
|
|
||||||
if (file->checksumPage)
|
if (file.checksumPage)
|
||||||
{
|
{
|
||||||
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, VARBOOL(!file->checksumPageError));
|
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, VARBOOL(!file.checksumPageError));
|
||||||
|
|
||||||
if (file->checksumPageErrorList != NULL)
|
if (file.checksumPageErrorList != NULL)
|
||||||
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, varNewVarLst(file->checksumPageErrorList));
|
kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, jsonToVar(file.checksumPageErrorList));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!varEq(manifestOwnerVar(file->group), saveData->groupDefault))
|
if (!varEq(manifestOwnerVar(file.group), saveData->groupDefault))
|
||||||
kvPut(fileKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(file->group));
|
kvPut(fileKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(file.group));
|
||||||
|
|
||||||
if (file->mode != saveData->fileModeDefault)
|
if (file.mode != saveData->fileModeDefault)
|
||||||
kvPut(fileKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", file->mode)));
|
kvPut(fileKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", file.mode)));
|
||||||
|
|
||||||
if (file->reference != NULL)
|
if (file.reference != NULL)
|
||||||
kvPut(fileKv, MANIFEST_KEY_REFERENCE_VAR, VARSTR(file->reference));
|
kvPut(fileKv, MANIFEST_KEY_REFERENCE_VAR, VARSTR(file.reference));
|
||||||
|
|
||||||
if (file->sizeRepo != file->size)
|
if (file.sizeRepo != file.size)
|
||||||
kvPut(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, VARUINT64(file->sizeRepo));
|
kvPut(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, VARUINT64(file.sizeRepo));
|
||||||
|
|
||||||
kvPut(fileKv, MANIFEST_KEY_SIZE_VAR, VARUINT64(file->size));
|
kvPut(fileKv, MANIFEST_KEY_SIZE_VAR, VARUINT64(file.size));
|
||||||
kvPut(fileKv, MANIFEST_KEY_TIMESTAMP_VAR, VARUINT64((uint64_t)file->timestamp));
|
kvPut(fileKv, MANIFEST_KEY_TIMESTAMP_VAR, VARUINT64((uint64_t)file.timestamp));
|
||||||
|
|
||||||
if (!varEq(manifestOwnerVar(file->user), saveData->userDefault))
|
if (!varEq(manifestOwnerVar(file.user), saveData->userDefault))
|
||||||
kvPut(fileKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(file->user));
|
kvPut(fileKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(file.user));
|
||||||
|
|
||||||
infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, file->name, jsonFromKv(fileKv));
|
infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, file.name, jsonFromKv(fileKv));
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_RESET(1000);
|
MEM_CONTEXT_TEMP_RESET(1000);
|
||||||
}
|
}
|
||||||
@ -2437,22 +2593,22 @@ manifestValidate(Manifest *this, bool strict)
|
|||||||
// Validate files
|
// Validate files
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFile(this, fileIdx);
|
const ManifestFile file = manifestFile(this, fileIdx);
|
||||||
|
|
||||||
// All files must have a checksum
|
// All files must have a checksum
|
||||||
if (file->checksumSha1[0] == '\0')
|
if (file.checksumSha1[0] == '\0')
|
||||||
strCatFmt(error, "\nmissing checksum for file '%s'", strZ(file->name));
|
strCatFmt(error, "\nmissing checksum for file '%s'", strZ(file.name));
|
||||||
|
|
||||||
// These are strict checks to be performed only after a backup and before the final manifest save
|
// These are strict checks to be performed only after a backup and before the final manifest save
|
||||||
if (strict)
|
if (strict)
|
||||||
{
|
{
|
||||||
// Zero-length files must have a specific checksum
|
// Zero-length files must have a specific checksum
|
||||||
if (file->size == 0 && !strEqZ(HASH_TYPE_SHA1_ZERO_STR, file->checksumSha1))
|
if (file.size == 0 && !strEqZ(HASH_TYPE_SHA1_ZERO_STR, file.checksumSha1))
|
||||||
strCatFmt(error, "\ninvalid checksum '%s' for zero size file '%s'", file->checksumSha1, strZ(file->name));
|
strCatFmt(error, "\ninvalid checksum '%s' for zero size file '%s'", file.checksumSha1, strZ(file.name));
|
||||||
|
|
||||||
// Non-zero size files must have non-zero repo size
|
// Non-zero size files must have non-zero repo size
|
||||||
if (file->sizeRepo == 0 && file->size != 0)
|
if (file.sizeRepo == 0 && file.size != 0)
|
||||||
strCatFmt(error, "\nrepo size must be > 0 for file '%s'", strZ(file->name));
|
strCatFmt(error, "\nrepo size must be > 0 for file '%s'", strZ(file.name));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2490,8 +2646,8 @@ manifestDbFind(const Manifest *this, const String *name)
|
|||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
File functions and getters/setters
|
File functions and getters/setters
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
const ManifestFile *
|
static ManifestFilePack **
|
||||||
manifestFileFind(const Manifest *this, const String *name)
|
manifestFilePackFindInternal(const Manifest *this, const String *name)
|
||||||
{
|
{
|
||||||
FUNCTION_TEST_BEGIN();
|
FUNCTION_TEST_BEGIN();
|
||||||
FUNCTION_TEST_PARAM(MANIFEST, this);
|
FUNCTION_TEST_PARAM(MANIFEST, this);
|
||||||
@ -2501,12 +2657,26 @@ manifestFileFind(const Manifest *this, const String *name)
|
|||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
ASSERT(name != NULL);
|
ASSERT(name != NULL);
|
||||||
|
|
||||||
const ManifestFile *result = lstFind(this->pub.fileList, &name);
|
ManifestFilePack **const filePack = lstFind(this->pub.fileList, &name);
|
||||||
|
|
||||||
if (result == NULL)
|
if (filePack == NULL)
|
||||||
THROW_FMT(AssertError, "unable to find '%s' in manifest file list", strZ(name));
|
THROW_FMT(AssertError, "unable to find '%s' in manifest file list", strZ(name));
|
||||||
|
|
||||||
FUNCTION_TEST_RETURN(result);
|
FUNCTION_TEST_RETURN(filePack);
|
||||||
|
}
|
||||||
|
|
||||||
|
const ManifestFilePack *
|
||||||
|
manifestFilePackFind(const Manifest *this, const String *name)
|
||||||
|
{
|
||||||
|
FUNCTION_TEST_BEGIN();
|
||||||
|
FUNCTION_TEST_PARAM(MANIFEST, this);
|
||||||
|
FUNCTION_TEST_PARAM(STRING, name);
|
||||||
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
|
ASSERT(this != NULL);
|
||||||
|
ASSERT(name != NULL);
|
||||||
|
|
||||||
|
FUNCTION_TEST_RETURN(*manifestFilePackFindInternal(this, name));
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
@ -2528,8 +2698,9 @@ manifestFileRemove(const Manifest *this, const String *name)
|
|||||||
|
|
||||||
void
|
void
|
||||||
manifestFileUpdate(
|
manifestFileUpdate(
|
||||||
Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference,
|
Manifest *const this, const String *const name, const uint64_t size, const uint64_t sizeRepo, const char *const checksumSha1,
|
||||||
bool checksumPage, bool checksumPageError, const VariantList *checksumPageErrorList)
|
const Variant *const reference, const bool checksumPage, const bool checksumPageError,
|
||||||
|
const String *const checksumPageErrorList)
|
||||||
{
|
{
|
||||||
FUNCTION_TEST_BEGIN();
|
FUNCTION_TEST_BEGIN();
|
||||||
FUNCTION_TEST_PARAM(MANIFEST, this);
|
FUNCTION_TEST_PARAM(MANIFEST, this);
|
||||||
@ -2540,7 +2711,7 @@ manifestFileUpdate(
|
|||||||
FUNCTION_TEST_PARAM(VARIANT, reference);
|
FUNCTION_TEST_PARAM(VARIANT, reference);
|
||||||
FUNCTION_TEST_PARAM(BOOL, checksumPage);
|
FUNCTION_TEST_PARAM(BOOL, checksumPage);
|
||||||
FUNCTION_TEST_PARAM(BOOL, checksumPageError);
|
FUNCTION_TEST_PARAM(BOOL, checksumPageError);
|
||||||
FUNCTION_TEST_PARAM(VARIANT_LIST, checksumPageErrorList);
|
FUNCTION_TEST_PARAM(STRING, checksumPageErrorList);
|
||||||
FUNCTION_TEST_END();
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
@ -2549,33 +2720,32 @@ manifestFileUpdate(
|
|||||||
(!checksumPage && !checksumPageError && checksumPageErrorList == NULL) ||
|
(!checksumPage && !checksumPageError && checksumPageErrorList == NULL) ||
|
||||||
(checksumPage && !checksumPageError && checksumPageErrorList == NULL) || (checksumPage && checksumPageError));
|
(checksumPage && !checksumPageError && checksumPageErrorList == NULL) || (checksumPage && checksumPageError));
|
||||||
|
|
||||||
ManifestFile *file = (ManifestFile *)manifestFileFind(this, name);
|
ManifestFilePack **const filePack = manifestFilePackFindInternal(this, name);
|
||||||
|
ManifestFile file = manifestFileUnpack(*filePack);
|
||||||
|
|
||||||
MEM_CONTEXT_BEGIN(lstMemContext(this->pub.fileList))
|
|
||||||
{
|
|
||||||
// Update reference if set
|
// Update reference if set
|
||||||
if (reference != NULL)
|
if (reference != NULL)
|
||||||
{
|
{
|
||||||
if (varStr(reference) == NULL)
|
if (varStr(reference) == NULL)
|
||||||
file->reference = NULL;
|
file.reference = NULL;
|
||||||
else
|
else
|
||||||
file->reference = strLstAddIfMissing(this->referenceList, varStr(reference));
|
file.reference = strLstAddIfMissing(this->referenceList, varStr(reference));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update checksum if set
|
// Update checksum if set
|
||||||
if (checksumSha1 != NULL)
|
if (checksumSha1 != NULL)
|
||||||
memcpy(file->checksumSha1, checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
|
memcpy(file.checksumSha1, checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1);
|
||||||
|
|
||||||
// Update repo size
|
// Update repo size
|
||||||
file->size = size;
|
file.size = size;
|
||||||
file->sizeRepo = sizeRepo;
|
file.sizeRepo = sizeRepo;
|
||||||
|
|
||||||
// Update checksum page info
|
// Update checksum page info
|
||||||
file->checksumPage = checksumPage;
|
file.checksumPage = checksumPage;
|
||||||
file->checksumPageError = checksumPageError;
|
file.checksumPageError = checksumPageError;
|
||||||
file->checksumPageErrorList = varLstDup(checksumPageErrorList);
|
file.checksumPageErrorList = checksumPageErrorList;
|
||||||
}
|
|
||||||
MEM_CONTEXT_END();
|
manifestFilePackUpdate(this, filePack, &file);
|
||||||
|
|
||||||
FUNCTION_TEST_RETURN_VOID();
|
FUNCTION_TEST_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
@ -96,7 +96,7 @@ typedef struct ManifestFile
|
|||||||
bool checksumPageError:1; // Is there an error in the page checksum?
|
bool checksumPageError:1; // Is there an error in the page checksum?
|
||||||
mode_t mode; // File mode
|
mode_t mode; // File mode
|
||||||
char checksumSha1[HASH_TYPE_SHA1_SIZE_HEX + 1]; // SHA1 checksum
|
char checksumSha1[HASH_TYPE_SHA1_SIZE_HEX + 1]; // SHA1 checksum
|
||||||
const VariantList *checksumPageErrorList; // List of page checksum errors if there are any
|
const String *checksumPageErrorList; // List of page checksum errors if there are any
|
||||||
const String *user; // User name
|
const String *user; // User name
|
||||||
const String *group; // Group name
|
const String *group; // Group name
|
||||||
const String *reference; // Reference to a prior backup
|
const String *reference; // Reference to a prior backup
|
||||||
@ -258,21 +258,52 @@ manifestDbTotal(const Manifest *const this)
|
|||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
File functions and getters/setters
|
File functions and getters/setters
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
__attribute__((always_inline)) static inline const ManifestFile *
|
typedef struct ManifestFilePack ManifestFilePack;
|
||||||
manifestFile(const Manifest *const this, const unsigned int fileIdx)
|
|
||||||
|
// Unpack file pack returned by manifestFilePackGet()
|
||||||
|
ManifestFile manifestFileUnpack(const ManifestFilePack *filePack);
|
||||||
|
|
||||||
|
// Get file in pack format by index
|
||||||
|
__attribute__((always_inline)) static inline const ManifestFilePack *
|
||||||
|
manifestFilePackGet(const Manifest *const this, const unsigned int fileIdx)
|
||||||
{
|
{
|
||||||
return lstGet(THIS_PUB(Manifest)->fileList, fileIdx);
|
return *(ManifestFilePack **)lstGet(THIS_PUB(Manifest)->fileList, fileIdx);
|
||||||
}
|
}
|
||||||
|
|
||||||
void manifestFileAdd(Manifest *this, const ManifestFile *file);
|
// Get file name
|
||||||
const ManifestFile *manifestFileFind(const Manifest *this, const String *name);
|
__attribute__((always_inline)) static inline const String *
|
||||||
|
manifestFileNameGet(const Manifest *const this, const unsigned int fileIdx)
|
||||||
|
{
|
||||||
|
return (const String *)manifestFilePackGet(this, fileIdx);
|
||||||
|
}
|
||||||
|
|
||||||
// If the file requested is not found in the list, return the default passed rather than throw an error
|
// Get file by index
|
||||||
__attribute__((always_inline)) static inline const ManifestFile *
|
__attribute__((always_inline)) static inline ManifestFile
|
||||||
manifestFileFindDefault(const Manifest *const this, const String *const name, const ManifestFile *const fileDefault)
|
manifestFile(const Manifest *const this, const unsigned int fileIdx)
|
||||||
|
{
|
||||||
|
return manifestFileUnpack(manifestFilePackGet(this, fileIdx));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add a file
|
||||||
|
void manifestFileAdd(Manifest *this, ManifestFile file);
|
||||||
|
|
||||||
|
// Find file in pack format by name
|
||||||
|
const ManifestFilePack *manifestFilePackFind(const Manifest *this, const String *name);
|
||||||
|
|
||||||
|
// Find file by name
|
||||||
|
__attribute__((always_inline)) static inline ManifestFile
|
||||||
|
manifestFileFind(const Manifest *const this, const String *const name)
|
||||||
{
|
{
|
||||||
ASSERT_INLINE(name != NULL);
|
ASSERT_INLINE(name != NULL);
|
||||||
return lstFindDefault(THIS_PUB(Manifest)->fileList, &name, (void *)fileDefault);
|
return manifestFileUnpack(manifestFilePackFind(this, name));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Does the file exist?
|
||||||
|
__attribute__((always_inline)) static inline bool
|
||||||
|
manifestFileExists(const Manifest *const this, const String *const name)
|
||||||
|
{
|
||||||
|
ASSERT_INLINE(name != NULL);
|
||||||
|
return lstFindDefault(THIS_PUB(Manifest)->fileList, &name, NULL) != NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
void manifestFileRemove(const Manifest *this, const String *name);
|
void manifestFileRemove(const Manifest *this, const String *name);
|
||||||
@ -286,7 +317,7 @@ manifestFileTotal(const Manifest *const this)
|
|||||||
// Update a file with new data
|
// Update a file with new data
|
||||||
void manifestFileUpdate(
|
void manifestFileUpdate(
|
||||||
Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference,
|
Manifest *this, const String *name, uint64_t size, uint64_t sizeRepo, const char *checksumSha1, const Variant *reference,
|
||||||
bool checksumPage, bool checksumPageError, const VariantList *checksumPageErrorList);
|
bool checksumPage, bool checksumPageError, const String *checksumPageErrorList);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Link functions and getters/setters
|
Link functions and getters/setters
|
||||||
|
@ -73,7 +73,7 @@ unit:
|
|||||||
|
|
||||||
# ----------------------------------------------------------------------------------------------------------------------------
|
# ----------------------------------------------------------------------------------------------------------------------------
|
||||||
- name: type-convert
|
- name: type-convert
|
||||||
total: 11
|
total: 12
|
||||||
|
|
||||||
coverage:
|
coverage:
|
||||||
- common/type/convert
|
- common/type/convert
|
||||||
|
@ -23,7 +23,7 @@ typedef struct TestBackupValidateCallbackData
|
|||||||
{
|
{
|
||||||
const Storage *storage; // Storage object when needed (e.g. fileCompressed = true)
|
const Storage *storage; // Storage object when needed (e.g. fileCompressed = true)
|
||||||
const String *path; // Subpath when storage is specified
|
const String *path; // Subpath when storage is specified
|
||||||
const Manifest *manifest; // Manifest to check for files/links/paths
|
Manifest *manifest; // Manifest to check for files/links/paths
|
||||||
const ManifestData *manifestData; // Manifest data
|
const ManifestData *manifestData; // Manifest data
|
||||||
String *content; // String where content should be added
|
String *content; // String where content should be added
|
||||||
} TestBackupValidateCallbackData;
|
} TestBackupValidateCallbackData;
|
||||||
@ -76,30 +76,31 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
|
|||||||
|
|
||||||
// Check against the manifest
|
// Check against the manifest
|
||||||
// ---------------------------------------------------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------------------------------------------------
|
||||||
const ManifestFile *file = manifestFileFind(data->manifest, manifestName);
|
ManifestFilePack **const filePack = manifestFilePackFindInternal(data->manifest, manifestName);
|
||||||
|
ManifestFile file = manifestFileUnpack(*filePack);
|
||||||
|
|
||||||
// Test size and repo-size. If compressed then set the repo-size to size so it will not be in test output. Even the same
|
// Test size and repo-size. If compressed then set the repo-size to size so it will not be in test output. Even the same
|
||||||
// compression algorithm can give slightly different results based on the version so repo-size is not deterministic for
|
// compression algorithm can give slightly different results based on the version so repo-size is not deterministic for
|
||||||
// compression.
|
// compression.
|
||||||
if (size != file->size)
|
if (size != file.size)
|
||||||
THROW_FMT(AssertError, "'%s' size does match manifest", strZ(manifestName));
|
THROW_FMT(AssertError, "'%s' size does match manifest", strZ(manifestName));
|
||||||
|
|
||||||
if (info->size != file->sizeRepo)
|
if (info->size != file.sizeRepo)
|
||||||
THROW_FMT(AssertError, "'%s' repo size does match manifest", strZ(manifestName));
|
THROW_FMT(AssertError, "'%s' repo size does match manifest", strZ(manifestName));
|
||||||
|
|
||||||
if (data->manifestData->backupOptionCompressType != compressTypeNone)
|
if (data->manifestData->backupOptionCompressType != compressTypeNone)
|
||||||
((ManifestFile *)file)->sizeRepo = file->size;
|
file.sizeRepo = file.size;
|
||||||
|
|
||||||
// Test the checksum. pg_control and WAL headers have different checksums depending on cpu architecture so remove
|
// Test the checksum. pg_control and WAL headers have different checksums depending on cpu architecture so remove
|
||||||
// the checksum from the test output.
|
// the checksum from the test output.
|
||||||
if (!strEqZ(checksum, file->checksumSha1))
|
if (!strEqZ(checksum, file.checksumSha1))
|
||||||
THROW_FMT(AssertError, "'%s' checksum does match manifest", strZ(manifestName));
|
THROW_FMT(AssertError, "'%s' checksum does match manifest", strZ(manifestName));
|
||||||
|
|
||||||
if (strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL) ||
|
if (strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL) ||
|
||||||
strBeginsWith(
|
strBeginsWith(
|
||||||
manifestName, strNewFmt(MANIFEST_TARGET_PGDATA "/%s/", strZ(pgWalPath(data->manifestData->pgVersion)))))
|
manifestName, strNewFmt(MANIFEST_TARGET_PGDATA "/%s/", strZ(pgWalPath(data->manifestData->pgVersion)))))
|
||||||
{
|
{
|
||||||
((ManifestFile *)file)->checksumSha1[0] = '\0';
|
file.checksumSha1[0] = '\0';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test mode, user, group. These values are not in the manifest but we know what they should be based on the default
|
// Test mode, user, group. These values are not in the manifest but we know what they should be based on the default
|
||||||
@ -113,6 +114,9 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
|
|||||||
if (!strEq(info->group, TEST_GROUP_STR))
|
if (!strEq(info->group, TEST_GROUP_STR))
|
||||||
THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(manifestName));
|
THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(manifestName));
|
||||||
|
|
||||||
|
// Update changes to manifest file
|
||||||
|
manifestFilePackUpdate(data->manifest, filePack, &file);
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1310,7 +1314,7 @@ testRun(void)
|
|||||||
|
|
||||||
manifestTargetAdd(manifestResume, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")});
|
manifestTargetAdd(manifestResume, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")});
|
||||||
manifestPathAdd(manifestResume, &(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR});
|
manifestPathAdd(manifestResume, &(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR});
|
||||||
manifestFileAdd(manifestResume, &(ManifestFile){.name = STRDEF("pg_data/" PG_FILE_PGVERSION)});
|
manifestFileAdd(manifestResume, (ManifestFile){.name = STRDEF("pg_data/" PG_FILE_PGVERSION)});
|
||||||
|
|
||||||
manifestSave(
|
manifestSave(
|
||||||
manifestResume,
|
manifestResume,
|
||||||
@ -1430,7 +1434,7 @@ testRun(void)
|
|||||||
OBJ_NEW_BEGIN(Manifest)
|
OBJ_NEW_BEGIN(Manifest)
|
||||||
{
|
{
|
||||||
manifest = manifestNewInternal();
|
manifest = manifestNewInternal();
|
||||||
manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF("pg_data/test")});
|
manifestFileAdd(manifest, (ManifestFile){.name = STRDEF("pg_data/test")});
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
@ -1748,9 +1752,12 @@ testRun(void)
|
|||||||
storagePg(), PG_FILE_PGVERSION, storageRepoWrite(),
|
storagePg(), PG_FILE_PGVERSION, storageRepoWrite(),
|
||||||
strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION", strZ(resumeLabel))));
|
strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION", strZ(resumeLabel))));
|
||||||
|
|
||||||
strcpy(
|
ManifestFilePack **const filePack = manifestFilePackFindInternal(manifestResume, STRDEF("pg_data/PG_VERSION"));
|
||||||
((ManifestFile *)manifestFileFind(manifestResume, STRDEF("pg_data/PG_VERSION")))->checksumSha1,
|
ManifestFile file = manifestFileUnpack(*filePack);
|
||||||
"06d06bb31b570b94d7b4325f511f853dbe771c21");
|
|
||||||
|
strcpy(file.checksumSha1, "06d06bb31b570b94d7b4325f511f853dbe771c21");
|
||||||
|
|
||||||
|
manifestFilePackUpdate(manifestResume, filePack, &file);
|
||||||
|
|
||||||
// Save the resume manifest
|
// Save the resume manifest
|
||||||
manifestSave(
|
manifestSave(
|
||||||
@ -1841,14 +1848,19 @@ testRun(void)
|
|||||||
HRN_STORAGE_PUT_EMPTY(
|
HRN_STORAGE_PUT_EMPTY(
|
||||||
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/global/pg_control.gz", strZ(resumeLabel))));
|
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/global/pg_control.gz", strZ(resumeLabel))));
|
||||||
|
|
||||||
((ManifestFile *)manifestFileFind(manifestResume, STRDEF("pg_data/global/pg_control")))->checksumSha1[0] = 0;
|
ManifestFilePack **const filePack = manifestFilePackFindInternal(manifestResume, STRDEF("pg_data/global/pg_control"));
|
||||||
|
ManifestFile file = manifestFileUnpack(*filePack);
|
||||||
|
|
||||||
|
file.checksumSha1[0] = 0;
|
||||||
|
|
||||||
|
manifestFilePackUpdate(manifestResume, filePack, &file);
|
||||||
|
|
||||||
// Size does not match between cluster and resume manifest
|
// Size does not match between cluster and resume manifest
|
||||||
HRN_STORAGE_PUT_Z(storagePgWrite(), "size-mismatch", "TEST", .timeModified = backupTimeStart);
|
HRN_STORAGE_PUT_Z(storagePgWrite(), "size-mismatch", "TEST", .timeModified = backupTimeStart);
|
||||||
HRN_STORAGE_PUT_EMPTY(
|
HRN_STORAGE_PUT_EMPTY(
|
||||||
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/size-mismatch.gz", strZ(resumeLabel))));
|
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/size-mismatch.gz", strZ(resumeLabel))));
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestResume, &(ManifestFile){
|
manifestResume, (ManifestFile){
|
||||||
.name = STRDEF("pg_data/size-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
|
.name = STRDEF("pg_data/size-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
|
||||||
.size = 33});
|
.size = 33});
|
||||||
|
|
||||||
@ -1857,7 +1869,7 @@ testRun(void)
|
|||||||
HRN_STORAGE_PUT_EMPTY(
|
HRN_STORAGE_PUT_EMPTY(
|
||||||
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch.gz", strZ(resumeLabel))));
|
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch.gz", strZ(resumeLabel))));
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestResume, &(ManifestFile){
|
manifestResume, (ManifestFile){
|
||||||
.name = STRDEF("pg_data/time-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", .size = 4,
|
.name = STRDEF("pg_data/time-mismatch"), .checksumSha1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", .size = 4,
|
||||||
.timestamp = backupTimeStart - 1});
|
.timestamp = backupTimeStart - 1});
|
||||||
|
|
||||||
@ -1867,7 +1879,7 @@ testRun(void)
|
|||||||
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/zero-size.gz", strZ(resumeLabel))),
|
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/zero-size.gz", strZ(resumeLabel))),
|
||||||
"ZERO-SIZE");
|
"ZERO-SIZE");
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestResume, &(ManifestFile){.name = STRDEF("pg_data/zero-size"), .size = 0, .timestamp = backupTimeStart});
|
manifestResume, (ManifestFile){.name = STRDEF("pg_data/zero-size"), .size = 0, .timestamp = backupTimeStart});
|
||||||
|
|
||||||
// Path is not in manifest
|
// Path is not in manifest
|
||||||
HRN_STORAGE_PATH_CREATE(
|
HRN_STORAGE_PATH_CREATE(
|
||||||
@ -2028,7 +2040,7 @@ testRun(void)
|
|||||||
HRN_STORAGE_PUT_EMPTY(
|
HRN_STORAGE_PUT_EMPTY(
|
||||||
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/resume-ref.gz", strZ(resumeLabel))));
|
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/resume-ref.gz", strZ(resumeLabel))));
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestResume, &(ManifestFile){.name = STRDEF("pg_data/resume-ref"), .size = 0, .reference = STRDEF("BOGUS")});
|
manifestResume, (ManifestFile){.name = STRDEF("pg_data/resume-ref"), .size = 0, .reference = STRDEF("BOGUS")});
|
||||||
|
|
||||||
// Time does not match between cluster and resume manifest (but resume because time is in future so delta enabled). Note
|
// Time does not match between cluster and resume manifest (but resume because time is in future so delta enabled). Note
|
||||||
// also that the repo file is intenionally corrupt to generate a warning about corruption in the repository.
|
// also that the repo file is intenionally corrupt to generate a warning about corruption in the repository.
|
||||||
@ -2036,7 +2048,7 @@ testRun(void)
|
|||||||
HRN_STORAGE_PUT_EMPTY(
|
HRN_STORAGE_PUT_EMPTY(
|
||||||
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch2.gz", strZ(resumeLabel))));
|
storageRepoWrite(), strZ(strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/time-mismatch2.gz", strZ(resumeLabel))));
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestResume, &(ManifestFile){
|
manifestResume, (ManifestFile){
|
||||||
.name = STRDEF("pg_data/time-mismatch2"), .checksumSha1 = "984816fd329622876e14907634264e6f332e9fb3", .size = 4,
|
.name = STRDEF("pg_data/time-mismatch2"), .checksumSha1 = "984816fd329622876e14907634264e6f332e9fb3", .size = 4,
|
||||||
.timestamp = backupTimeStart});
|
.timestamp = backupTimeStart});
|
||||||
|
|
||||||
|
@ -131,7 +131,7 @@ testManifestMinimal(const String *label, unsigned int pgVersion, const String *p
|
|||||||
manifestPathAdd(result, &pathBase);
|
manifestPathAdd(result, &pathBase);
|
||||||
ManifestFile fileVersion = {
|
ManifestFile fileVersion = {
|
||||||
.name = STRDEF("pg_data/" PG_FILE_PGVERSION), .mode = 0600, .group = groupName(), .user = userName()};
|
.name = STRDEF("pg_data/" PG_FILE_PGVERSION), .mode = 0600, .group = groupName(), .user = userName()};
|
||||||
manifestFileAdd(result, &fileVersion);
|
manifestFileAdd(result, fileVersion);
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
@ -815,7 +815,7 @@ testRun(void)
|
|||||||
|
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/pg_hba.conf"), .size = 4, .timestamp = 1482182860});
|
(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/pg_hba.conf"), .size = 4, .timestamp = 1482182860});
|
||||||
|
|
||||||
TEST_RESULT_VOID(restoreManifestMap(manifest), "remap links");
|
TEST_RESULT_VOID(restoreManifestMap(manifest), "remap links");
|
||||||
|
|
||||||
@ -982,7 +982,7 @@ testRun(void)
|
|||||||
ManifestPath path = {.name = STRDEF("pg_data/bogus_path"), .user = STRDEF("path-user-bogus")};
|
ManifestPath path = {.name = STRDEF("pg_data/bogus_path"), .user = STRDEF("path-user-bogus")};
|
||||||
manifestPathAdd(manifest, &path);
|
manifestPathAdd(manifest, &path);
|
||||||
ManifestFile file = {.name = STRDEF("pg_data/bogus_file"), .mode = 0600, .group = STRDEF("file-group-bogus")};
|
ManifestFile file = {.name = STRDEF("pg_data/bogus_file"), .mode = 0600, .group = STRDEF("file-group-bogus")};
|
||||||
manifestFileAdd(manifest, &file);
|
manifestFileAdd(manifest, file);
|
||||||
ManifestLink link = {.name = STRDEF("pg_data/bogus_link"), .destination = STRDEF("/"), .group = STRDEF("link-group-bogus")};
|
ManifestLink link = {.name = STRDEF("pg_data/bogus_link"), .destination = STRDEF("/"), .group = STRDEF("link-group-bogus")};
|
||||||
manifestLinkAdd(manifest, &link);
|
manifestLinkAdd(manifest, &link);
|
||||||
|
|
||||||
@ -1034,7 +1034,7 @@ testRun(void)
|
|||||||
|
|
||||||
manifest = testManifestMinimal(STRDEF("20161219-212741F_20161219-21275D"), PG_VERSION_96, pgPath);
|
manifest = testManifestMinimal(STRDEF("20161219-212741F_20161219-21275D"), PG_VERSION_96, pgPath);
|
||||||
|
|
||||||
manifestFileAdd(manifest, &file);
|
manifestFileAdd(manifest, file);
|
||||||
manifestLinkAdd(manifest, &link);
|
manifestLinkAdd(manifest, &link);
|
||||||
|
|
||||||
TEST_RESULT_VOID(restoreManifestOwner(manifest, &rootReplaceUser, &rootReplaceGroup), "check ownership");
|
TEST_RESULT_VOID(restoreManifestOwner(manifest, &rootReplaceUser, &rootReplaceGroup), "check ownership");
|
||||||
@ -1051,7 +1051,7 @@ testRun(void)
|
|||||||
TEST_TITLE("owner is root and ownership of pg_data is bad");
|
TEST_TITLE("owner is root and ownership of pg_data is bad");
|
||||||
|
|
||||||
manifestPathAdd(manifest, &path);
|
manifestPathAdd(manifest, &path);
|
||||||
manifestFileAdd(manifest, &file);
|
manifestFileAdd(manifest, file);
|
||||||
|
|
||||||
HRN_SYSTEM_FMT("sudo chown 77777:77777 %s", strZ(pgPath));
|
HRN_SYSTEM_FMT("sudo chown 77777:77777 %s", strZ(pgPath));
|
||||||
|
|
||||||
@ -1243,7 +1243,7 @@ testRun(void)
|
|||||||
|
|
||||||
HRN_SYSTEM_FMT("rm -rf %s/*", strZ(pgPath));
|
HRN_SYSTEM_FMT("rm -rf %s/*", strZ(pgPath));
|
||||||
|
|
||||||
manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)});
|
manifestFileAdd(manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_POSTGRESQLAUTOCONF)});
|
||||||
|
|
||||||
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_POSTGRESQLAUTOCONF);
|
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_POSTGRESQLAUTOCONF);
|
||||||
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_RECOVERYSIGNAL);
|
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), PG_FILE_RECOVERYSIGNAL);
|
||||||
@ -1303,7 +1303,7 @@ testRun(void)
|
|||||||
manifest->pub.data.pgCatalogVersion = hrnPgCatalogVersion(PG_VERSION_90);
|
manifest->pub.data.pgCatalogVersion = hrnPgCatalogVersion(PG_VERSION_90);
|
||||||
|
|
||||||
manifestTargetAdd(manifest, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")});
|
manifestTargetAdd(manifest, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = STRDEF("/pg")});
|
||||||
manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION)});
|
manifestFileAdd(manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION)});
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
@ -1324,11 +1324,11 @@ testRun(void)
|
|||||||
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("user-made-system-db"), .id = 16380, .lastSystemId = 12168});
|
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("user-made-system-db"), .id = 16380, .lastSystemId = 12168});
|
||||||
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF(UTF8_DB_NAME), .id = 16384, .lastSystemId = 12168});
|
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF(UTF8_DB_NAME), .id = 16384, .lastSystemId = 12168});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/1/" PG_FILE_PGVERSION)});
|
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/1/" PG_FILE_PGVERSION)});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16381/" PG_FILE_PGVERSION)});
|
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16381/" PG_FILE_PGVERSION)});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16385/" PG_FILE_PGVERSION)});
|
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16385/" PG_FILE_PGVERSION)});
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_END();
|
MEM_CONTEXT_END();
|
||||||
|
|
||||||
@ -1353,7 +1353,7 @@ testRun(void)
|
|||||||
MEM_CONTEXT_BEGIN(manifest->pub.memContext)
|
MEM_CONTEXT_BEGIN(manifest->pub.memContext)
|
||||||
{
|
{
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16384/" PG_FILE_PGVERSION)});
|
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/16384/" PG_FILE_PGVERSION)});
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_END();
|
MEM_CONTEXT_END();
|
||||||
|
|
||||||
@ -1424,7 +1424,7 @@ testRun(void)
|
|||||||
{
|
{
|
||||||
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test2"), .id = 32768, .lastSystemId = 12168});
|
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test2"), .id = 32768, .lastSystemId = 12168});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)});
|
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)});
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_END();
|
MEM_CONTEXT_END();
|
||||||
|
|
||||||
@ -1448,7 +1448,7 @@ testRun(void)
|
|||||||
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387"), .tablespaceId = 16387, .tablespaceName = STRDEF("ts1"),
|
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387"), .tablespaceId = 16387, .tablespaceName = STRDEF("ts1"),
|
||||||
.path = STRDEF("/ts1")});
|
.path = STRDEF("/ts1")});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest, &(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)});
|
manifest, (ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_BASE "/32768/" PG_FILE_PGVERSION)});
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_END();
|
MEM_CONTEXT_END();
|
||||||
|
|
||||||
@ -1470,7 +1470,7 @@ testRun(void)
|
|||||||
{
|
{
|
||||||
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test3"), .id = 65536, .lastSystemId = 12168});
|
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("test3"), .id = 65536, .lastSystemId = 12168});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest, &(ManifestFile){
|
manifest, (ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387/PG_9.4_201409291/65536/" PG_FILE_PGVERSION)});
|
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/16387/PG_9.4_201409291/65536/" PG_FILE_PGVERSION)});
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_END();
|
MEM_CONTEXT_END();
|
||||||
@ -2053,7 +2053,7 @@ testRun(void)
|
|||||||
// PG_VERSION
|
// PG_VERSION
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"});
|
.checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"});
|
||||||
@ -2184,7 +2184,7 @@ testRun(void)
|
|||||||
// tablespace_map (will be ignored during restore)
|
// tablespace_map (will be ignored during restore)
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
|
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
|
||||||
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP);
|
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP);
|
||||||
@ -2218,7 +2218,7 @@ testRun(void)
|
|||||||
// pg_tblspc/1/16384/PG_VERSION
|
// pg_tblspc/1/16384/PG_VERSION
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/1/16384/" PG_FILE_PGVERSION), .size = 4,
|
.name = STRDEF(MANIFEST_TARGET_PGTBLSPC "/1/16384/" PG_FILE_PGVERSION), .size = 4,
|
||||||
.timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(),
|
.timestamp = 1482182860, .mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"});
|
.checksumSha1 = "b74d60e763728399bcd3fb63f7dd1f97b46c6b44"});
|
||||||
@ -2400,7 +2400,7 @@ testRun(void)
|
|||||||
|
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL), .size = 8192, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL), .size = 8192, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "5e2b96c19c4f5c63a5afa2de504d29fe64a4c908"});
|
.checksumSha1 = "5e2b96c19c4f5c63a5afa2de504d29fe64a4c908"});
|
||||||
@ -2409,7 +2409,7 @@ testRun(void)
|
|||||||
// global/999
|
// global/999
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/999"), .size = 0, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/999"), .size = 0, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = HASH_TYPE_SHA1_ZERO, .reference = STRDEF(TEST_LABEL)});
|
.checksumSha1 = HASH_TYPE_SHA1_ZERO, .reference = STRDEF(TEST_LABEL)});
|
||||||
@ -2418,7 +2418,7 @@ testRun(void)
|
|||||||
// PG_VERSION
|
// PG_VERSION
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
|
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
|
||||||
@ -2439,7 +2439,7 @@ testRun(void)
|
|||||||
// base/1/PG_VERSION
|
// base/1/PG_VERSION
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA "base/1/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA "base/1/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
|
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
|
||||||
@ -2452,7 +2452,7 @@ testRun(void)
|
|||||||
|
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA "base/1/2"), .size = 8192, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA "base/1/2"), .size = 8192, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "4d7b2a36c5387decf799352a3751883b7ceb96aa"});
|
.checksumSha1 = "4d7b2a36c5387decf799352a3751883b7ceb96aa"});
|
||||||
@ -2470,7 +2470,7 @@ testRun(void)
|
|||||||
// base/16384/PG_VERSION
|
// base/16384/PG_VERSION
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA "base/16384/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA "base/16384/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
|
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
|
||||||
@ -2483,7 +2483,7 @@ testRun(void)
|
|||||||
|
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA "base/16384/16385"), .size = 16384, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA "base/16384/16385"), .size = 16384, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "d74e5f7ebe52a3ed468ba08c5b6aefaccd1ca88f"});
|
.checksumSha1 = "d74e5f7ebe52a3ed468ba08c5b6aefaccd1ca88f"});
|
||||||
@ -2499,7 +2499,7 @@ testRun(void)
|
|||||||
// base/32768/PG_VERSION
|
// base/32768/PG_VERSION
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA "base/32768/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA "base/32768/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
|
.checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
|
||||||
@ -2512,7 +2512,7 @@ testRun(void)
|
|||||||
|
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA "base/32768/32769"), .size = 32768, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA "base/32768/32769"), .size = 32768, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "a40f0986acb1531ce0cc75a23dcf8aa406ae9081"});
|
.checksumSha1 = "a40f0986acb1531ce0cc75a23dcf8aa406ae9081"});
|
||||||
@ -2529,7 +2529,7 @@ testRun(void)
|
|||||||
.name = name, .destination = STRDEF("../config/postgresql.conf"), .group = groupName(), .user = userName()});
|
.name = name, .destination = STRDEF("../config/postgresql.conf"), .group = groupName(), .user = userName()});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA "postgresql.conf"), .size = 15, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA "postgresql.conf"), .size = 15, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "98b8abb2e681e2a5a7d8ab082c0a79727887558d"});
|
.checksumSha1 = "98b8abb2e681e2a5a7d8ab082c0a79727887558d"});
|
||||||
@ -2546,7 +2546,7 @@ testRun(void)
|
|||||||
.name = name, .destination = STRDEF("../config/pg_hba.conf"), .group = groupName(), .user = userName()});
|
.name = name, .destination = STRDEF("../config/pg_hba.conf"), .group = groupName(), .user = userName()});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA "pg_hba.conf"), .size = 11, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA "pg_hba.conf"), .size = 11, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(),
|
.mode = 0600, .group = groupName(), .user = userName(),
|
||||||
.checksumSha1 = "401215e092779574988a854d8c7caed7f91dba4b"});
|
.checksumSha1 = "401215e092779574988a854d8c7caed7f91dba4b"});
|
||||||
@ -2555,7 +2555,7 @@ testRun(void)
|
|||||||
// tablespace_map (will be ignored during restore)
|
// tablespace_map (will be ignored during restore)
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860,
|
.name = STRDEF(TEST_PGDATA PG_FILE_TABLESPACEMAP), .size = 0, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
|
.mode = 0600, .group = groupName(), .user = userName(), .checksumSha1 = HASH_TYPE_SHA1_ZERO});
|
||||||
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP);
|
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_FILE_TABLESPACEMAP);
|
||||||
|
@ -202,5 +202,36 @@ testRun(void)
|
|||||||
TEST_RESULT_UINT(cvtZToUInt64("18446744073709551615"), 18446744073709551615U, "convert string to uint64");
|
TEST_RESULT_UINT(cvtZToUInt64("18446744073709551615"), 18446744073709551615U, "convert string to uint64");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// *****************************************************************************************************************************
|
||||||
|
if (testBegin("cvtUInt64ToVarInt128() and cvtUInt64FromVarInt128()"))
|
||||||
|
{
|
||||||
|
uint8_t buffer[CVT_VARINT128_BUFFER_SIZE];
|
||||||
|
size_t bufferPos;
|
||||||
|
|
||||||
|
bufferPos = 0;
|
||||||
|
TEST_ERROR(cvtUInt64ToVarInt128(9999, buffer, &bufferPos, 1), AssertError, "buffer overflow");
|
||||||
|
|
||||||
|
bufferPos = 0;
|
||||||
|
TEST_RESULT_VOID(cvtUInt64ToVarInt128(0xFFFFFFFFFFFFFFFF, buffer, &bufferPos, sizeof(buffer)), "to varint-128");
|
||||||
|
TEST_RESULT_UINT(bufferPos, 10, "check buffer position");
|
||||||
|
|
||||||
|
bufferPos = 0;
|
||||||
|
TEST_RESULT_UINT(cvtUInt64FromVarInt128(buffer, &bufferPos), 0xFFFFFFFFFFFFFFFF, "to uint64");
|
||||||
|
TEST_RESULT_UINT(bufferPos, 10, "check buffer position");
|
||||||
|
|
||||||
|
bufferPos = 0;
|
||||||
|
TEST_RESULT_VOID(cvtUInt64ToVarInt128(0, buffer, &bufferPos, sizeof(buffer)), "to varint-128");
|
||||||
|
TEST_RESULT_UINT(bufferPos, 1, "check buffer position");
|
||||||
|
|
||||||
|
bufferPos = 0;
|
||||||
|
TEST_RESULT_UINT(cvtUInt64FromVarInt128(buffer, &bufferPos), 0, "to uint64");
|
||||||
|
TEST_RESULT_UINT(bufferPos, 1, "check buffer position");
|
||||||
|
|
||||||
|
uint8_t buffer2[CVT_VARINT128_BUFFER_SIZE + 1];
|
||||||
|
memset(buffer2, 0xFF, sizeof(buffer2));
|
||||||
|
bufferPos = 0;
|
||||||
|
TEST_ERROR(cvtUInt64FromVarInt128(buffer2, &bufferPos), FormatError, "unterminated varint-128 integer");
|
||||||
|
}
|
||||||
|
|
||||||
FUNCTION_HARNESS_RETURN_VOID();
|
FUNCTION_HARNESS_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
@ -950,7 +950,7 @@ testRun(void)
|
|||||||
|
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860});
|
(ManifestFile){.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .timestamp = 1482182860});
|
||||||
|
|
||||||
TEST_RESULT_VOID(manifestBuildValidate(manifest, false, 1482182860, false), "validate manifest");
|
TEST_RESULT_VOID(manifestBuildValidate(manifest, false, 1482182860, false), "validate manifest");
|
||||||
TEST_RESULT_INT(manifest->pub.data.backupTimestampCopyStart, 1482182860, "check copy start");
|
TEST_RESULT_INT(manifest->pub.data.backupTimestampCopyStart, 1482182860, "check copy start");
|
||||||
@ -1029,22 +1029,22 @@ testRun(void)
|
|||||||
&(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR, .mode = 0700, .group = STRDEF("test"), .user = STRDEF("test")});
|
&(ManifestPath){.name = MANIFEST_TARGET_PGDATA_STR, .mode = 0700, .group = STRDEF("test"), .user = STRDEF("test")});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/BOGUS"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/BOGUS"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182861,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182861,
|
||||||
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
||||||
}
|
}
|
||||||
@ -1059,17 +1059,17 @@ testRun(void)
|
|||||||
|
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestPrior,
|
manifestPrior,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE3"), .size = 0, .sizeRepo = 0, .timestamp = 1482182860,
|
||||||
.checksumSha1 = "da39a3ee5e6b4b0d3255bfef95601890afd80709"});
|
.checksumSha1 = "da39a3ee5e6b4b0d3255bfef95601890afd80709"});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestPrior,
|
manifestPrior,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE4"), .size = 55, .sizeRepo = 55, .timestamp = 1482182860,
|
||||||
.checksumSha1 = "ccccccccccaaaaaaaaaabbbbbbbbbbdddddddddd"});
|
.checksumSha1 = "ccccccccccaaaaaaaaaabbbbbbbbbbdddddddddd"});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestPrior,
|
manifestPrior,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
||||||
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
|
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
|
||||||
}
|
}
|
||||||
@ -1109,18 +1109,18 @@ testRun(void)
|
|||||||
lstClear(manifest->pub.fileList);
|
lstClear(manifest->pub.fileList);
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/" PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
||||||
|
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestPrior,
|
manifestPrior,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
||||||
.reference = STRDEF("20190101-010101F_20190202-010101D"),
|
.reference = STRDEF("20190101-010101F_20190202-010101D"),
|
||||||
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
|
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd"});
|
||||||
@ -1160,7 +1160,7 @@ testRun(void)
|
|||||||
|
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182859,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182859,
|
||||||
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
||||||
|
|
||||||
@ -1171,11 +1171,11 @@ testRun(void)
|
|||||||
varLstAdd(checksumPageErrorList, varNewUInt(77));
|
varLstAdd(checksumPageErrorList, varNewUInt(77));
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestPrior,
|
manifestPrior,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
||||||
.reference = STRDEF("20190101-010101F_20190202-010101D"),
|
.reference = STRDEF("20190101-010101F_20190202-010101D"),
|
||||||
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", .checksumPage = true, .checksumPageError = true,
|
.checksumSha1 = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", .checksumPage = true, .checksumPageError = true,
|
||||||
.checksumPageErrorList = checksumPageErrorList});
|
.checksumPageErrorList = jsonFromVar(varNewVarLst(checksumPageErrorList))});
|
||||||
|
|
||||||
TEST_RESULT_VOID(manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, NULL), "incremental manifest");
|
TEST_RESULT_VOID(manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, NULL), "incremental manifest");
|
||||||
|
|
||||||
@ -1211,18 +1211,18 @@ testRun(void)
|
|||||||
lstClear(manifest->pub.fileList);
|
lstClear(manifest->pub.fileList);
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
|
||||||
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 6, .sizeRepo = 6, .timestamp = 1482182860,
|
||||||
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
||||||
|
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestPrior,
|
manifestPrior,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
||||||
.reference = STRDEF("20190101-010101F_20190202-010101D"),
|
.reference = STRDEF("20190101-010101F_20190202-010101D"),
|
||||||
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
|
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
|
||||||
@ -1279,14 +1279,14 @@ testRun(void)
|
|||||||
lstClear(manifest->pub.fileList);
|
lstClear(manifest->pub.fileList);
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifest,
|
manifest,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE1"), .size = 6, .sizeRepo = 6, .timestamp = 1482182861,
|
||||||
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
.mode = 0600, .group = STRDEF("test"), .user = STRDEF("test")});
|
||||||
|
|
||||||
manifest->pub.data.backupOptionOnline = BOOL_TRUE_VAR;
|
manifest->pub.data.backupOptionOnline = BOOL_TRUE_VAR;
|
||||||
manifestFileAdd(
|
manifestFileAdd(
|
||||||
manifestPrior,
|
manifestPrior,
|
||||||
&(ManifestFile){
|
(ManifestFile){
|
||||||
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
.name = STRDEF(MANIFEST_TARGET_PGDATA "/FILE2"), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
|
||||||
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
|
.checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa"});
|
||||||
|
|
||||||
@ -1729,18 +1729,14 @@ testRun(void)
|
|||||||
TEST_TITLE("manifest getters");
|
TEST_TITLE("manifest getters");
|
||||||
|
|
||||||
// ManifestFile getters
|
// ManifestFile getters
|
||||||
const ManifestFile *file = NULL;
|
ManifestFile file = {0};
|
||||||
TEST_ERROR(manifestFileFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest file list");
|
TEST_ERROR(manifestFileFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest file list");
|
||||||
TEST_ASSIGN(file, manifestFileFind(manifest, STRDEF("pg_data/PG_VERSION")), "manifestFileFind()");
|
TEST_ASSIGN(file, manifestFileFind(manifest, STRDEF("pg_data/PG_VERSION")), "manifestFileFind()");
|
||||||
TEST_RESULT_STR_Z(file->name, "pg_data/PG_VERSION", "find file");
|
TEST_RESULT_STR_Z(file.name, "pg_data/PG_VERSION", "find file");
|
||||||
TEST_RESULT_STR_Z(
|
TEST_RESULT_STR_Z(
|
||||||
manifestFileFindDefault(manifest, STRDEF("bogus"), file)->name, "pg_data/PG_VERSION",
|
manifestFileFind(manifest, STRDEF("pg_data/special-@#!$^&*()_+~`{}[]\\:;")).name,
|
||||||
"manifestFileFindDefault() - return default");
|
|
||||||
TEST_RESULT_STR_Z(
|
|
||||||
manifestFileFind(manifest, STRDEF("pg_data/special-@#!$^&*()_+~`{}[]\\:;"))->name,
|
|
||||||
"pg_data/special-@#!$^&*()_+~`{}[]\\:;", "find special file");
|
"pg_data/special-@#!$^&*()_+~`{}[]\\:;", "find special file");
|
||||||
TEST_ASSIGN(file, manifestFileFindDefault(manifest, STRDEF("bogus"), NULL), "manifestFileFindDefault()");
|
TEST_RESULT_BOOL(manifestFileExists(manifest, STRDEF("bogus")), false, "manifest file does not exist");
|
||||||
TEST_RESULT_PTR(file, NULL, "return default NULL");
|
|
||||||
|
|
||||||
TEST_RESULT_VOID(
|
TEST_RESULT_VOID(
|
||||||
manifestFileUpdate(manifest, STRDEF("pg_data/postgresql.conf"), 4457, 4457, "", NULL, false, false, NULL),
|
manifestFileUpdate(manifest, STRDEF("pg_data/postgresql.conf"), 4457, 4457, "", NULL, false, false, NULL),
|
||||||
|
@ -317,8 +317,8 @@ testRun(void)
|
|||||||
|
|
||||||
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
|
||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFile(manifest, fileIdx);
|
const ManifestFile file = manifestFile(manifest, fileIdx);
|
||||||
ASSERT(file == manifestFileFind(manifest, file->name));
|
ASSERT(strEq(file.name, manifestFileFind(manifest, file.name).name));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
|
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
|
||||||
|
Reference in New Issue
Block a user