diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 87a125820..29c5a852d 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -48,6 +48,15 @@

Partial multi-repository implementation.

+ + + + + + +

Add backup verification to internal verify command.

+ + diff --git a/src/command/verify/verify.c b/src/command/verify/verify.c index 1f92e6cbb..2f7df9751 100644 --- a/src/command/verify/verify.c +++ b/src/command/verify/verify.c @@ -5,8 +5,9 @@ Verify the contents of the repository. ***********************************************************************************************************************************/ #include "build.auto.h" -#include #include +#include +#include #include "command/archive/common.h" #include "command/check/common.h" @@ -36,11 +37,17 @@ Data Types and Structures #define FUNCTION_LOG_VERIFY_ARCHIVE_RESULT_FORMAT(value, buffer, bufferSize) \ objToLog(&value, "VerifyArchiveResult", buffer, bufferSize) +#define FUNCTION_LOG_VERIFY_BACKUP_RESULT_TYPE \ + VerifyBackupResult +#define FUNCTION_LOG_VERIFY_BACKUP_RESULT_FORMAT(value, buffer, bufferSize) \ + objToLog(&value, "VerifyBackupResult", buffer, bufferSize) + // Structure for verifying repository info files typedef struct VerifyInfoFile { InfoBackup *backup; // Backup.info file contents InfoArchive *archive; // Archive.info file contents + Manifest *manifest; // Manifest file contents const String *checksum; // File checksum int errorCode; // Error code else 0 for no error } VerifyInfoFile; @@ -70,6 +77,31 @@ typedef struct VerifyInvalidFile VerifyResult reason; // Reason file is invalid (e.g. incorrect checksum) } VerifyInvalidFile; +// Status result of a backup +typedef enum +{ + backupValid, // Default: All files in backup label repo passed verification + backupInvalid, // One of more files in backup label repo failed verification + backupMissingManifest, // Backup manifest missing (backup may have expired) + backupInProgress, // Backup appeared to be in progress (so was skipped) +} VerifyBackupResultStatus; + +typedef struct VerifyBackupResult +{ + String *backupLabel; // Label assigned to the backup + VerifyBackupResultStatus status; // Final status of the backup + bool fileVerifyComplete; // Have all the files of the backup completed verification? + unsigned int totalFileManifest; // Total number of backup files in the manifest + unsigned int totalFileVerify; // Total number of backup files being verified + unsigned int totalFileValid; // Total number of backup files that were verified and valid + String *backupPrior; // Prior backup that this backup depends on, if any + unsigned int pgId; // PG id will be used to find WAL for the backup in the repo + unsigned int pgVersion; // PG version will be used with PG id to find WAL in the repo + String *archiveStart; // First WAL segment in the backup + String *archiveStop; // Last WAL segment in the backup + List *invalidFileList; // List of invalid files found in the backup +} VerifyBackupResult; + // Job data stucture for processing and results collection typedef struct VerifyJobData { @@ -78,14 +110,49 @@ typedef struct VerifyJobData StringList *walPathList; // WAL path list for a single archive id StringList *walFileList; // WAL file list for a single WAL path StringList *backupList; // List of backups to verify + Manifest *manifest; // Manifest contents with list of files to verify + unsigned int manifestFileIdx; // Index of the file within the manifest file list to process String *currentBackup; // In progress backup, if any const InfoPg *pgHistory; // Database history list - bool backupProcessing; // Are we processing WAL or are we processing backup + bool backupProcessing; // Are we processing WAL or are we processing backups + const String *manifestCipherPass; // Cipher pass for reading backup manifests const String *walCipherPass; // Cipher pass for reading WAL files + const String *backupCipherPass; // Cipher pass for reading backup files referenced in a manifest unsigned int jobErrorTotal; // Total errors that occurred during the job execution List *archiveIdResultList; // Archive results + List *backupResultList; // Backup results } VerifyJobData; +/*********************************************************************************************************************************** +Helper function to add a file to an invalid file list +***********************************************************************************************************************************/ +static void +verifyInvalidFileAdd(List *invalidFileList, VerifyResult reason, const String *fileName) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(LIST, invalidFileList); // Invalid file list to add the filename to + FUNCTION_TEST_PARAM(ENUM, reason); // Reason for invalid file + FUNCTION_TEST_PARAM(STRING, fileName); // Name of invalid file + FUNCTION_TEST_END(); + + ASSERT(invalidFileList != NULL); + ASSERT(fileName != NULL); + + MEM_CONTEXT_BEGIN(lstMemContext(invalidFileList)) + { + VerifyInvalidFile invalidFile = + { + .fileName = strDup(fileName), + .reason = reason, + }; + + lstAdd(invalidFileList, &invalidFile); + } + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + /*********************************************************************************************************************************** Load a file into memory ***********************************************************************************************************************************/ @@ -142,8 +209,10 @@ verifyInfoFile(const String *pathFileName, bool keepFile, const String *cipherPa { if (strBeginsWith(pathFileName, INFO_BACKUP_PATH_FILE_STR)) result.backup = infoBackupMove(infoBackupNewLoad(infoRead), memContextPrior()); - else + else if (strBeginsWith(pathFileName, INFO_ARCHIVE_PATH_FILE_STR)) result.archive = infoArchiveMove(infoArchiveNewLoad(infoRead), memContextPrior()); + else + result.manifest = manifestMove(manifestNewLoad(infoRead), memContextPrior()); } else ioReadDrain(infoRead); @@ -277,6 +346,136 @@ verifyBackupInfoFile(void) FUNCTION_LOG_RETURN(INFO_BACKUP, result); } +/*********************************************************************************************************************************** +Get the manifest file +***********************************************************************************************************************************/ +static Manifest * +verifyManifestFile( + VerifyBackupResult *backupResult, const String *cipherPass, bool currentBackup, const InfoPg *pgHistory, + unsigned int *jobErrorTotal) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_TEST_PARAM_P(VERIFY_BACKUP_RESULT, backupResult); // The result set for the backup being processed + FUNCTION_TEST_PARAM(STRING, cipherPass); // Passphrase to access the manifest file + FUNCTION_LOG_PARAM(BOOL, currentBackup); // Is this possibly a backup currently in progress? + FUNCTION_TEST_PARAM(INFO_PG, pgHistory); // Database history + FUNCTION_TEST_PARAM_P(UINT, jobErrorTotal); // Pointer to the overall job error total + FUNCTION_LOG_END(); + + Manifest *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + String *fileName = strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(backupResult->backupLabel)); + + // Get the main manifest file + VerifyInfoFile verifyManifestInfo = verifyInfoFile(fileName, true, cipherPass); + + // If the main file did not error, then report on the copy's status and check checksums + if (verifyManifestInfo.errorCode == 0) + { + result = verifyManifestInfo.manifest; + + // The current in-progress backup is only notional until the main file is checked because the backup may have + // completed by the time the main manifest is checked here. So having a main manifest file means this backup is not + // (or is no longer) the currentBackup. + currentBackup = false; + + // Attempt to load the copy and report on it's status but don't keep it in memory + VerifyInfoFile verifyManifestInfoCopy = verifyInfoFile( + strNewFmt("%s%s", strZ(fileName), INFO_COPY_EXT), false, cipherPass); + + // If the copy loaded successfuly, then check the checksums + if (verifyManifestInfoCopy.errorCode == 0) + { + // If the manifest and manifest.copy checksums don't match each other than one (or both) of the files could be + // corrupt so log a warning but trust main + if (!strEq(verifyManifestInfo.checksum, verifyManifestInfoCopy.checksum)) + LOG_WARN_FMT("backup '%s' manifest.copy does not match manifest", strZ(backupResult->backupLabel)); + } + } + else + { + // If this might be an in-progress backup and the main manifest is simply missing, it is assumed the backup is an + // actual in-progress backup and verification is skipped, otherwise, if the main is not simply missing, or this is not + // an in-progress backup then attempt to load the copy. + if (!(currentBackup && verifyManifestInfo.errorCode == errorTypeCode(&FileMissingError))) + { + currentBackup = false; + + VerifyInfoFile verifyManifestInfoCopy = verifyInfoFile( + strNewFmt("%s%s", strZ(fileName), INFO_COPY_EXT), true, cipherPass); + + // If loaded successfully, then return the copy as usable + if (verifyManifestInfoCopy.errorCode == 0) + { + LOG_WARN_FMT("%s/backup.manifest is missing or unusable, using copy", strZ(backupResult->backupLabel)); + + result = verifyManifestInfoCopy.manifest; + } + else if (verifyManifestInfo.errorCode == errorTypeCode(&FileMissingError) && + verifyManifestInfoCopy.errorCode == errorTypeCode(&FileMissingError)) + { + backupResult->status = backupMissingManifest; + + LOG_WARN_FMT("manifest missing for '%s' - backup may have expired", strZ(backupResult->backupLabel)); + } + } + else + { + backupResult->status = backupInProgress; + + LOG_INFO_FMT("backup '%s' appears to be in progress, skipping", strZ(backupResult->backupLabel)); + } + } + + // If found a usable manifest then check that the database it was based on is in the history + if (result != NULL) + { + bool found = false; + const ManifestData *manData = manifestData(result); + + // Confirm the PG database information from the manifest is in the history list + for (unsigned int infoPgIdx = 0; infoPgIdx < infoPgDataTotal(pgHistory); infoPgIdx++) + { + InfoPgData pgHistoryData = infoPgData(pgHistory, infoPgIdx); + + if (pgHistoryData.id == manData->pgId && pgHistoryData.systemId == manData->pgSystemId && + pgHistoryData.version == manData->pgVersion) + { + found = true; + break; + } + } + + // If the PG data is not found in the backup.info history, then error and reset the result + if (!found) + { + LOG_ERROR_FMT( + errorTypeCode(&FileInvalidError), + "'%s' may not be recoverable - PG data (id %u, version %s, system-id %" PRIu64 ") is not in the backup.info" + " history, skipping", + strZ(backupResult->backupLabel), manData->pgId, strZ(pgVersionToStr(manData->pgVersion)), manData->pgSystemId); + + manifestFree(result); + result = NULL; + } + else + manifestMove(result, memContextPrior()); + } + + // If the result is NULL and the backup status has not yet been set, then the backup is unusable (invalid) + if (result == NULL && backupResult->status == backupValid) + { + backupResult->status = backupInvalid; + (*jobErrorTotal)++; + } + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(MANIFEST, result); +} + /*********************************************************************************************************************************** Check the history in the info files ***********************************************************************************************************************************/ @@ -411,7 +610,7 @@ verifyCreateArchiveIdRange(VerifyArchiveResult *archiveIdResult, StringList *wal { .start = strDup(walSegment), .stop = strDup(walSegment), - .invalidFileList = lstNewP(sizeof(VerifyInvalidFile), .comparator = lstComparatorStr), + .invalidFileList = lstNewP(sizeof(VerifyInvalidFile), .comparator = lstComparatorStr), }; lstAdd(archiveIdResult->walRangeList, &walRangeNew); @@ -470,7 +669,7 @@ verifyArchive(void *data) VerifyArchiveResult archiveIdResult = { .archiveId = strDup(archiveId), - .walRangeList = lstNewP(sizeof(VerifyWalRange), .comparator = lstComparatorStr), + .walRangeList = lstNewP(sizeof(VerifyWalRange), .comparator = lstComparatorStr), }; lstAdd(jobData->archiveIdResultList, &archiveIdResult); @@ -561,8 +760,9 @@ verifyArchive(void *data) protocolCommandParamAdd(command, VARUINT64(archiveResult->pgWalInfo.size)); protocolCommandParamAdd(command, VARSTR(jobData->walCipherPass)); - // Assign job to result - result = protocolParallelJobNew(VARSTR(filePathName), command); + // Assign job to result, prepending the archiveId to the key for consistency with backup processing + result = protocolParallelJobNew( + VARSTR(strNewFmt("%s/%s", strZ(archiveResult->archiveId), strZ(filePathName))), command); // Remove the file to process from the list strLstRemoveIdx(jobData->walFileList, 0); @@ -610,6 +810,229 @@ verifyArchive(void *data) FUNCTION_TEST_RETURN(result); } +/*********************************************************************************************************************************** +Verify the job data backups +***********************************************************************************************************************************/ +static ProtocolParallelJob * +verifyBackup(void *data) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM_P(VOID, data); + FUNCTION_TEST_END(); + + ProtocolParallelJob *result = NULL; + + VerifyJobData *jobData = data; + + // Process backup files, if any + while (strLstSize(jobData->backupList) > 0) + { + result = NULL; + + // If result list is empty or the last processed is not equal to the backup being processed, then intialize the backup + // data and results + if (lstSize(jobData->backupResultList) == 0 || + !strEq(((VerifyBackupResult *)lstGetLast(jobData->backupResultList))->backupLabel, strLstGet(jobData->backupList, 0))) + { + MEM_CONTEXT_BEGIN(lstMemContext(jobData->backupResultList)) + { + VerifyBackupResult backupResultNew = + { + .backupLabel = strDup(strLstGet(jobData->backupList, 0)), + .invalidFileList = lstNewP(sizeof(VerifyInvalidFile), .comparator = lstComparatorStr), + }; + + // Add the backup to the result list + lstAdd(jobData->backupResultList, &backupResultNew); + } + MEM_CONTEXT_END(); + + // Get the result just added so it can be updated directly + VerifyBackupResult *backupResult = lstGetLast(jobData->backupResultList); + + // If currentBackup is set (meaning the newest backup label on disk was not in the db:current section when the + // backup.info file was read) and this is the same label, then set inProgessBackup to true, else false. + // inProgressBackup may be changed in verifyManifestFile if a main backup.manifest exists since that would indicate the + // backup completed during the verify process. + bool inProgressBackup = strEq(jobData->currentBackup, backupResult->backupLabel); + + // Get a usable backup manifest file + Manifest *manifest = verifyManifestFile( + backupResult, jobData->manifestCipherPass, inProgressBackup, jobData->pgHistory, &jobData->jobErrorTotal); + + // If a usable backup.manifest file is not found + if (manifest == NULL) + { + // Remove this backup from the processing list + strLstRemoveIdx(jobData->backupList, 0); + + // No files to process so continue to the next backup in the list + continue; + } + // Initialize the backup results and manifest for processing + else + { + // Move the manifest to the jobData for processing + jobData->manifest = manifestMove(manifest, jobData->memContext); + + // Initialize the jobData + MEM_CONTEXT_BEGIN(jobData->memContext) + { + // Get the cipher subpass used to decrypt files in the backup and initialize the file list index + jobData->backupCipherPass = strDup(manifestCipherSubPass(jobData->manifest)); + jobData->manifestFileIdx = 0; + } + MEM_CONTEXT_END(); + + const ManifestData *manData = manifestData(jobData->manifest); + + MEM_CONTEXT_BEGIN(lstMemContext(jobData->backupResultList)) + { + backupResult->totalFileManifest = manifestFileTotal(jobData->manifest); + backupResult->backupPrior = strDup(manData->backupLabelPrior); + backupResult->pgId = manData->pgId; + backupResult->pgVersion = manData->pgVersion; + backupResult->archiveStart = strDup(manData->archiveStart); + backupResult->archiveStop = strDup(manData->archiveStop); + } + MEM_CONTEXT_END(); + } + } + + VerifyBackupResult *backupResult = lstGetLast(jobData->backupResultList); + + // Process any files in the manifest + if (jobData->manifestFileIdx < manifestFileTotal(jobData->manifest)) + { + do + { + const ManifestFile *fileData = manifestFile(jobData->manifest, jobData->manifestFileIdx); + + String *filePathName = NULL; + + // Track the files verified in order to determine when the processing of the backup is complete + backupResult->totalFileVerify++; + + // Check if the file is referenced in a prior backup + if (fileData->reference != NULL) + { + // If the prior backup is not in the result list, then that backup was never processed (likely due to the --set + // option) so verify the file + unsigned int backupPriorIdx = lstFindIdx(jobData->backupResultList, &fileData->reference); + + if (backupPriorIdx == LIST_NOT_FOUND) + { + filePathName = strNewFmt( + STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name), + strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); + } + // Else the backup this file references has a result so check the processing state for the referenced backup + else + { + VerifyBackupResult *backupResultPrior = lstGet(jobData->backupResultList, backupPriorIdx); + + // If the verify-state of the backup is not complete then verify the file + if (!backupResultPrior->fileVerifyComplete) + { + filePathName = strNewFmt( + STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name), + strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); + } + // Else skip verification + else + { + String *priorFile = strNewFmt( + "%s/%s%s", strZ(fileData->reference), strZ(fileData->name), + strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); + + unsigned int backupPriorInvalidIdx = lstFindIdx(backupResultPrior->invalidFileList, &priorFile); + + // If the file is in the invalid file list of the prior backup where it is referenced then add the file + // as invalid to this backup result and set the backup result status; since already logged an error on + // this file, don't log again + if (backupPriorInvalidIdx != LIST_NOT_FOUND) + { + VerifyInvalidFile *invalidFile = lstGet( + backupResultPrior->invalidFileList, backupPriorInvalidIdx); + verifyInvalidFileAdd(backupResult->invalidFileList, invalidFile->reason, invalidFile->fileName); + backupResult->status = backupInvalid; + } + // Else the file in the prior backup was valid so increment the total valid files for this backup + else + { + backupResult->totalFileValid++; + } + } + } + } + // Else file is not referenced in a prior backup + else + { + filePathName = strNewFmt( + STORAGE_REPO_BACKUP "/%s/%s%s", strZ(backupResult->backupLabel), strZ(fileData->name), + strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType))); + } + + // If constructed file name is not null then send it off for processing + if (filePathName != NULL) + { + // Set up the job + ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE_STR); + protocolCommandParamAdd(command, VARSTR(filePathName)); + + // If the checksum is not present in the manifest, it will be calculated by manifest load + protocolCommandParamAdd(command, VARSTRZ(fileData->checksumSha1)); + protocolCommandParamAdd(command, VARUINT64(fileData->size)); + protocolCommandParamAdd(command, VARSTR(jobData->backupCipherPass)); + + // Assign job to result (prepend backup label being processed to the key since some files are in a prior backup) + result = protocolParallelJobNew( + VARSTR(strNewFmt("%s/%s", strZ(backupResult->backupLabel), strZ(filePathName))), command); + } + + // Increment the index to point to the next file + jobData->manifestFileIdx++; + + // If this was the last file to process for this backup, then free the manifest and remove this backup from the + // processing list + if (jobData->manifestFileIdx == backupResult->totalFileManifest) + { + manifestFree(jobData->manifest); + jobData->manifest = NULL; + strLstRemoveIdx(jobData->backupList, 0); + } + + // If a job was found to be processed then break out to process it + if (result != NULL) + break; + } + while (jobData->manifestFileIdx < backupResult->totalFileManifest); + } + else + { + // Nothing to process so report an error, free the manifest, set the status, and remove the backup from processing list + LOG_ERROR_FMT( + errorTypeCode(&FileInvalidError), "backup '%s' manifest does not contain any target files to verify", + strZ(backupResult->backupLabel)); + + jobData->jobErrorTotal++; + + manifestFree(jobData->manifest); + jobData->manifest = NULL; + + backupResult->status = backupInvalid; + + strLstRemoveIdx(jobData->backupList, 0); + } + + // If a job was found to be processed then break out to process it + if (result != NULL) + break; + } + + FUNCTION_TEST_RETURN(result); +} + /*********************************************************************************************************************************** Process the job data ***********************************************************************************************************************************/ @@ -626,7 +1049,6 @@ verifyJobCallback(void *data, unsigned int clientIdx) // Initialize the result ProtocolParallelJob *result = NULL; - // Get a new job if there are any left MEM_CONTEXT_TEMP_BEGIN() { VerifyJobData *jobData = data; @@ -634,8 +1056,17 @@ verifyJobCallback(void *data, unsigned int clientIdx) if (!jobData->backupProcessing) { result = protocolParallelJobMove(verifyArchive(data), memContextPrior()); + + // Set the backupProcessing flag if the archive processing is finished so backup processing can begin immediately after jobData->backupProcessing = strLstSize(jobData->archiveIdList) == 0; } + + if (jobData->backupProcessing) + { + // Only begin backup verification if the last archive result was processed + if (result == NULL) + result = protocolParallelJobMove(verifyBackup(data), memContextPrior()); + } } MEM_CONTEXT_TEMP_END(); @@ -661,7 +1092,7 @@ verifyErrorMsg(VerifyResult verifyResult) else if (verifyResult == verifySizeInvalid) result = strCatZ(result, "invalid size"); else - result = strCatZ(result, "invalid verify"); + result = strCatZ(result, "invalid result"); FUNCTION_TEST_RETURN(result); } @@ -670,19 +1101,21 @@ verifyErrorMsg(VerifyResult verifyResult) Helper function to output a log message based on job result that is not verifyOk and return an error count ***********************************************************************************************************************************/ static unsigned int -verifyLogInvalidResult(VerifyResult verifyResult, unsigned int processId, String *filePathName) +verifyLogInvalidResult(const String *fileType, VerifyResult verifyResult, unsigned int processId, const String *filePathName) { FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, fileType); // Indicates archive or backup file FUNCTION_TEST_PARAM(ENUM, verifyResult); // Result code from the verifyFile() function FUNCTION_TEST_PARAM(UINT, processId); // Process Id reporting the result FUNCTION_TEST_PARAM(STRING, filePathName); // File for which results are being reported FUNCTION_TEST_END(); + ASSERT(fileType != NULL); ASSERT(filePathName != NULL); // Log a warning because the WAL may have gone missing if expire came through and removed it // legitimately so it is not necessarily an error so the jobErrorTotal should not be incremented - if (verifyResult == verifyFileMissing) + if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR) && verifyResult == verifyFileMissing) { LOG_WARN_PID_FMT(processId, "%s '%s'", strZ(verifyErrorMsg(verifyResult)), strZ(filePathName)); FUNCTION_TEST_RETURN(0); @@ -787,25 +1220,23 @@ verifyAddInvalidWalFile(List *walRangeList, VerifyResult fileResult, String *fil ASSERT(fileName != NULL); ASSERT(walSegment != NULL); - for (unsigned int walIdx = 0; walIdx < lstSize(walRangeList); walIdx++) + MEM_CONTEXT_TEMP_BEGIN() { - VerifyWalRange *walRange = lstGet(walRangeList, walIdx); - - // If the WAL segment is less/equal to the stop file then it falls in this range since ranges are sorted by stop file in - // ascending order, therefore first one found is the range - if (strCmp(walRange->stop, walSegment) >= 0) + for (unsigned int walIdx = 0; walIdx < lstSize(walRangeList); walIdx++) { - VerifyInvalidFile invalidFile = - { - .fileName = strDup(fileName), - .reason = fileResult, - }; + VerifyWalRange *walRange = lstGet(walRangeList, walIdx); - // Add the file to the range where it was found and exit the loop - lstAdd(walRange->invalidFileList, &invalidFile); - break; + // If the WAL segment is less/equal to the stop file then it falls in this range since ranges are sorted by stop file in + // ascending order, therefore first one found is the range + if (strCmp(walRange->stop, walSegment) >= 0) + { + // Add the file to the range where it was found and exit the loop + verifyInvalidFileAdd(walRange->invalidFileList, fileResult, fileName); + break; + } } } + MEM_CONTEXT_TEMP_END(); FUNCTION_TEST_RETURN_VOID(); } @@ -814,43 +1245,122 @@ verifyAddInvalidWalFile(List *walRangeList, VerifyResult fileResult, String *fil Render the results of the verify command ***********************************************************************************************************************************/ static String * -verifyRender(List *archiveIdResultList) +verifyRender(List *archiveIdResultList, List *backupResultList) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(LIST, archiveIdResultList); // Result list for all archive Ids in the repo + FUNCTION_TEST_PARAM(LIST, backupResultList); // Result list for all backups in the repo FUNCTION_TEST_END(); ASSERT(archiveIdResultList != NULL); + ASSERT(backupResultList != NULL); - String *result = strNew("Results:\n"); + String *result = strNew("Results:"); - for (unsigned int archiveIdx = 0; archiveIdx < lstSize(archiveIdResultList); archiveIdx++) + // Render archive results + if (lstSize(archiveIdResultList) == 0) + strCatZ(result, "\n archiveId: none found"); + else { - VerifyArchiveResult *archiveIdResult = lstGet(archiveIdResultList, archiveIdx); - strCatFmt( - result, "%s archiveId: %s, total WAL checked: %u, total valid WAL: %u", (archiveIdx > 0 ? "\n" : ""), - strZ(archiveIdResult->archiveId), archiveIdResult->totalWalFile, archiveIdResult->totalValidWal); - - if (archiveIdResult->totalWalFile > 0) + for (unsigned int archiveIdx = 0; archiveIdx < lstSize(archiveIdResultList); archiveIdx++) { - unsigned int errMissing = 0; - unsigned int errChecksum = 0; - unsigned int errSize = 0; - unsigned int errOther = 0; + VerifyArchiveResult *archiveIdResult = lstGet(archiveIdResultList, archiveIdx); + strCatFmt( + result, "\n archiveId: %s, total WAL checked: %u, total valid WAL: %u", strZ(archiveIdResult->archiveId), + archiveIdResult->totalWalFile, archiveIdResult->totalValidWal); - for (unsigned int walIdx = 0; walIdx < lstSize(archiveIdResult->walRangeList); walIdx++) + if (archiveIdResult->totalWalFile > 0) { - VerifyWalRange *walRange = lstGet(archiveIdResult->walRangeList, walIdx); + unsigned int errMissing = 0; + unsigned int errChecksum = 0; + unsigned int errSize = 0; + unsigned int errOther = 0; - LOG_DETAIL_FMT( - "archiveId: %s, wal start: %s, wal stop: %s", strZ(archiveIdResult->archiveId), strZ(walRange->start), - strZ(walRange->stop)); - - unsigned int invalidIdx = 0; - - while (invalidIdx < lstSize(walRange->invalidFileList)) + for (unsigned int walIdx = 0; walIdx < lstSize(archiveIdResult->walRangeList); walIdx++) { - VerifyInvalidFile *invalidFile = lstGet(walRange->invalidFileList, invalidIdx); + VerifyWalRange *walRange = lstGet(archiveIdResult->walRangeList, walIdx); + + LOG_DETAIL_FMT( + "archiveId: %s, wal start: %s, wal stop: %s", strZ(archiveIdResult->archiveId), strZ(walRange->start), + strZ(walRange->stop)); + + unsigned int invalidIdx = 0; + + while (invalidIdx < lstSize(walRange->invalidFileList)) + { + VerifyInvalidFile *invalidFile = lstGet(walRange->invalidFileList, invalidIdx); + + if (invalidFile->reason == verifyFileMissing) + errMissing++; + else if (invalidFile->reason == verifyChecksumMismatch) + errChecksum++; + else if (invalidFile->reason == verifySizeInvalid) + errSize++; + else + errOther++; + + invalidIdx++; + } + } + + strCatFmt( + result, "\n missing: %u, checksum invalid: %u, size invalid: %u, other: %u", errMissing, errChecksum, + errSize, errOther); + } + } + } + + // Render backup results + if (lstSize(backupResultList) == 0) + strCatZ(result, "\n backup: none found"); + else + { + for (unsigned int backupIdx = 0; backupIdx < lstSize(backupResultList); backupIdx++) + { + VerifyBackupResult *backupResult = lstGet(backupResultList, backupIdx); + String *status = NULL; + + switch (backupResult->status) + { + case backupValid: + { + status = strNew("valid"); + break; + } + + case backupInvalid: + { + status = strNew("invalid"); + break; + } + + case backupMissingManifest: + { + status = strNew("manifest missing"); + break; + } + + case backupInProgress: + { + status = strNew("in-progress"); + break; + } + } + + strCatFmt( + result, "\n backup: %s, status: %s, total files checked: %u, total valid files: %u", + strZ(backupResult->backupLabel), strZ(status), backupResult->totalFileVerify, backupResult->totalFileValid); + + if (backupResult->totalFileVerify > 0) + { + unsigned int errMissing = 0; + unsigned int errChecksum = 0; + unsigned int errSize = 0; + unsigned int errOther = 0; + + for (unsigned int invalidIdx = 0; invalidIdx < lstSize(backupResult->invalidFileList); invalidIdx++) + { + VerifyInvalidFile *invalidFile = lstGet(backupResult->invalidFileList, invalidIdx); if (invalidFile->reason == verifyFileMissing) errMissing++; @@ -860,15 +1370,12 @@ verifyRender(List *archiveIdResultList) errSize++; else errOther++; - - invalidIdx++; } - } - strCatFmt( - result, - "\n missing: %u, checksum invalid: %u, size invalid: %u, other: %u", - errMissing, errChecksum, errSize, errOther); + strCatFmt( + result, "\n missing: %u, checksum invalid: %u, size invalid: %u, other: %u", errMissing, errChecksum, + errSize, errOther); + } } } @@ -940,18 +1447,20 @@ verifyProcess(unsigned int *errorTotal) .walPathList = NULL, .walFileList = strLstNew(), .pgHistory = infoArchivePg(archiveInfo), + .manifestCipherPass = infoPgCipherPass(infoBackupPg(backupInfo)), .walCipherPass = infoPgCipherPass(infoArchivePg(archiveInfo)), - .archiveIdResultList = lstNewP(sizeof(VerifyArchiveResult), .comparator = archiveIdComparator), + .archiveIdResultList = lstNewP(sizeof(VerifyArchiveResult), .comparator = archiveIdComparator), + .backupResultList = lstNewP(sizeof(VerifyBackupResult), .comparator = lstComparatorStr), }; - // Get a list of backups in the repo + // Get a list of backups in the repo sorted ascending jobData.backupList = strLstSort( storageListP( storage, STORAGE_REPO_BACKUP_STR, .expression = backupRegExpP(.full = true, .differential = true, .incremental = true)), sortOrderAsc); - // Get a list of archive Ids in the repo (e.g. 9.4-1, 10-2, etc) sorted by the db-id (number after the dash) + // Get a list of archive Ids in the repo (e.g. 9.4-1, 10-2, etc) sorted ascending by the db-id (number after the dash) jobData.archiveIdList = strLstSort( strLstComparatorSet( storageListP(storage, STORAGE_REPO_ARCHIVE_STR, .expression = STRDEF(REGEX_ARCHIVE_DIR_DB_VERSION)), @@ -966,6 +1475,10 @@ verifyProcess(unsigned int *errorTotal) if (strLstSize(jobData.archiveIdList) == 0 || strLstSize(jobData.backupList) == 0) LOG_WARN_FMT("no %s exist in the repo", strLstSize(jobData.archiveIdList) == 0 ? "archives" : "backups"); + // If there are no archives to process, then set the processing flag to skip to processing the backups + if (strLstSize(jobData.archiveIdList) == 0) + jobData.backupProcessing = true; + // Set current backup if there is one and verify the archive history on disk is in the database history jobData.currentBackup = verifySetBackupCheckArchive( jobData.backupList, backupInfo, jobData.archiveIdList, jobData.pgHistory, &jobData.jobErrorTotal); @@ -989,33 +1502,68 @@ verifyProcess(unsigned int *errorTotal) ProtocolParallelJob *job = protocolParallelResult(parallelExec); unsigned int processId = protocolParallelJobProcessId(job); StringList *filePathLst = strLstNewSplit(varStr(protocolParallelJobKey(job)), FSLASH_STR); + + // Remove the result and file type identifier and recreate the path file name + const String *resultId = strLstGet(filePathLst, 0); + strLstRemoveIdx(filePathLst, 0); + const String *fileType = strLstGet(filePathLst, 0); strLstRemoveIdx(filePathLst, 0); String *filePathName = strLstJoin(filePathLst, "/"); + // Initialize the result sets VerifyArchiveResult *archiveIdResult = NULL; + VerifyBackupResult *backupResult = NULL; - // Find the archiveId in the list - assert if not found since this should never happen - String *archiveId = strLstGet(filePathLst, 0); - unsigned int index = lstFindIdx(jobData.archiveIdResultList, &archiveId); - ASSERT(index != LIST_NOT_FOUND); + // Get archiveId result data + if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR)) + { + // Find the archiveId in the list - assert if not found since this should never happen + unsigned int index = lstFindIdx(jobData.archiveIdResultList, &resultId); + ASSERT(index != LIST_NOT_FOUND); - archiveIdResult = lstGet(jobData.archiveIdResultList, index); + archiveIdResult = lstGet(jobData.archiveIdResultList, index); + } + // Else get the backup result data + else + { + unsigned int index = lstFindIdx(jobData.backupResultList, &resultId); + ASSERT(index != LIST_NOT_FOUND); + + backupResult = lstGet(jobData.backupResultList, index); + } // The job was successful if (protocolParallelJobErrorCode(job) == 0) { const VerifyResult verifyResult = (VerifyResult)varUIntForce(protocolParallelJobResult(job)); - if (verifyResult == verifyOk) - archiveIdResult->totalValidWal++; + // Update the result set for the type of file being processed + if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR)) + { + if (verifyResult == verifyOk) + archiveIdResult->totalValidWal++; + else + { + jobData.jobErrorTotal += verifyLogInvalidResult( + fileType, verifyResult, processId, filePathName); + + // Add invalid file to the WAL range + verifyAddInvalidWalFile( + archiveIdResult->walRangeList, verifyResult, filePathName, + strSubN(strLstGet(filePathLst, strLstSize(filePathLst) - 1), 0, WAL_SEGMENT_NAME_SIZE)); + } + } else { - jobData.jobErrorTotal += verifyLogInvalidResult(verifyResult, processId, filePathName); - - // Add invalid file with reason from result of verifyFile to range list - verifyAddInvalidWalFile( - archiveIdResult->walRangeList, verifyResult, filePathName, - strSubN(strLstGet(filePathLst, strLstSize(filePathLst) - 1), 0, WAL_SEGMENT_NAME_SIZE)); + if (verifyResult == verifyOk) + backupResult->totalFileValid++; + else + { + jobData.jobErrorTotal += verifyLogInvalidResult( + fileType, verifyResult, processId, filePathName); + backupResult->status = backupInvalid; + verifyInvalidFileAdd(backupResult->invalidFileList, verifyResult, filePathName); + } } } // Else the job errored @@ -1029,10 +1577,26 @@ verifyProcess(unsigned int *errorTotal) jobData.jobErrorTotal++; - // Add invalid file with "OtherError" reason to range list - verifyAddInvalidWalFile( - archiveIdResult->walRangeList, verifyOtherError, filePathName, - strSubN(strLstGet(filePathLst, strLstSize(filePathLst) - 1), 0, WAL_SEGMENT_NAME_SIZE)); + // Add invalid file with "OtherError" reason to invalid file list + if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR)) + { + // Add invalid file to the WAL range + verifyAddInvalidWalFile( + archiveIdResult->walRangeList, verifyOtherError, filePathName, + strSubN(strLstGet(filePathLst, strLstSize(filePathLst) - 1), 0, WAL_SEGMENT_NAME_SIZE)); + } + else + { + backupResult->status = backupInvalid; + verifyInvalidFileAdd(backupResult->invalidFileList, verifyOtherError, filePathName); + } + } + + // Set backup verification complete for a backup if all files have run through verification + if (strEq(fileType, STORAGE_REPO_BACKUP_STR) && + backupResult->totalFileVerify == backupResult->totalFileManifest) + { + backupResult->fileVerifyComplete = true; } // Free the job @@ -1044,8 +1608,7 @@ verifyProcess(unsigned int *errorTotal) // ??? Need to do the final reconciliation - checking backup required WAL against, valid WAL // Report results - if (lstSize(jobData.archiveIdResultList) > 0) - resultStr = verifyRender(jobData.archiveIdResultList); + resultStr = verifyRender(jobData.archiveIdResultList, jobData.backupResultList); } else LOG_WARN("no archives or backups exist in the repo"); diff --git a/test/define.yaml b/test/define.yaml index 32aab2ea1..9809a476b 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -712,7 +712,7 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: verify - total: 6 + total: 8 binReq: true coverage: diff --git a/test/src/module/command/verifyTest.c b/test/src/module/command/verifyTest.c index 7f552a688..e7b273ee0 100644 --- a/test/src/module/command/verifyTest.c +++ b/test/src/module/command/verifyTest.c @@ -33,68 +33,88 @@ testRun(void) strLstAdd(argListBase, strNewFmt("--stanza=%s", strZ(stanza))); strLstAdd(argListBase, strNewFmt("--repo1-path=%s/repo", testPath())); + const char *fileContents = "acefile"; + uint64_t fileSize = 7; + const String *fileChecksum = STRDEF("d1cd8a7d11daa26814b93eb604e1d49ab4b43770"); + + #define TEST_BACKUP_DB1_94 \ + "db-catalog-version=201409291\n" \ + "db-control-version=942\n" \ + "db-id=1\n" \ + "db-system-id=6625592122879095702\n" \ + "db-version=\"9.4\"\n" + + #define TEST_BACKUP_DB2_11 \ + "db-catalog-version=201707211\n" \ + "db-control-version=1100\n" \ + "db-id=2\n" \ + "db-system-id=6626363367545678089\n" \ + "db-version=\"11\"\n" + + #define TEST_BACKUP_DB1_CURRENT_FULL1 \ + "20181119-152138F={" \ + "\"backrest-format\":5,\"backrest-version\":\"2.28dev\"," \ + "\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\"," \ + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \ + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \ + "\"backup-timestamp-start\":1482182846,\"backup-timestamp-stop\":1482182861,\"backup-type\":\"full\"," \ + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \ + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + + #define TEST_BACKUP_DB1_CURRENT_FULL2 \ + "20181119-152800F={" \ + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," \ + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \ + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \ + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," \ + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \ + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + + #define TEST_BACKUP_DB1_CURRENT_FULL3 \ + "20181119-152900F={" \ + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," \ + "\"backup-archive-start\":\"000000010000000000000004\",\"backup-archive-stop\":\"000000010000000000000004\"," \ + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \ + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \ + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," \ + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \ + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + + #define TEST_BACKUP_DB1_HISTORY \ + "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," \ + "\"db-version\":\"9.4\"}" + + #define TEST_BACKUP_DB2_HISTORY \ + "2={\"db-catalog-version\":201707211,\"db-control-version\":1100,\"db-system-id\":6626363367545678089," \ + "\"db-version\":\"11\"}" + String *backupInfoContent = strNewFmt( "[backup:current]\n" - "20181119-152138F={" - "\"backrest-format\":5,\"backrest-version\":\"2.28dev\"," - "\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\"," - "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," - "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," - "\"backup-timestamp-start\":1482182846,\"backup-timestamp-stop\":1482182861,\"backup-type\":\"full\"," - "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," - "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + TEST_BACKUP_DB1_CURRENT_FULL1 "\n" "[db]\n" - "db-catalog-version=201409291\n" - "db-control-version=942\n" - "db-id=1\n" - "db-system-id=6625592122879095702\n" - "db-version=\"9.4\"\n" + TEST_BACKUP_DB1_94 "\n" "[db:history]\n" - "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," - "\"db-version\":\"9.4\"}"); + TEST_BACKUP_DB1_HISTORY + ); const Buffer *backupInfoBase = harnessInfoChecksumZ(strZ(backupInfoContent)); String *backupInfoMultiHistoryContent = strNewFmt( "[backup:current]\n" - "20181119-152138F={" - "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," - "\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\"," - "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," - "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," - "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," - "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," - "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" - "20181119-152800F={" - "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," - "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," - "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," - "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," - "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," - "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" - "20181119-152900F={" - "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," - "\"backup-archive-start\":\"000000010000000000000004\",\"backup-archive-stop\":\"000000010000000000000004\"," - "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," - "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," - "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," - "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," - "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + TEST_BACKUP_DB1_CURRENT_FULL1 + TEST_BACKUP_DB1_CURRENT_FULL2 + TEST_BACKUP_DB1_CURRENT_FULL3 "\n" "[db]\n" - "db-catalog-version=201707211\n" - "db-control-version=1100\n" - "db-id=2\n" - "db-system-id=6626363367545678089\n" - "db-version=\"11\"\n" + TEST_BACKUP_DB2_11 "\n" "[db:history]\n" - "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," - "\"db-version\":\"9.4\"}\n" - "2={\"db-catalog-version\":201707211,\"db-control-version\":1100,\"db-system-id\":6626363367545678089," - "\"db-version\":\"11\"}"); + TEST_BACKUP_DB1_HISTORY + "\n" + TEST_BACKUP_DB2_HISTORY + ); const Buffer *backupInfoMultiHistoryBase = harnessInfoChecksumZ(strZ(backupInfoMultiHistoryContent)); @@ -121,6 +141,288 @@ testRun(void) const Buffer *archiveInfoMultiHistoryBase = harnessInfoChecksumZ(strZ(archiveInfoMultiHistoryContent)); + #define TEST_MANIFEST_HEADER \ + "[backup]\n" \ + "backup-label=null\n" \ + "backup-timestamp-copy-start=0\n" \ + "backup-timestamp-start=0\n" \ + "backup-timestamp-stop=0\n" \ + "backup-type=\"full\"\n" + + #define TEST_MANIFEST_DB_92 \ + "\n" \ + "[backup:db]\n" \ + "db-catalog-version=201204301\n" \ + "db-control-version=922\n" \ + "db-id=1\n" \ + "db-system-id=6625592122879095702\n" \ + "db-version=\"9.2\"\n" + + #define TEST_MANIFEST_DB_94 \ + "\n" \ + "[backup:db]\n" \ + "db-catalog-version=201409291\n" \ + "db-control-version=942\n" \ + "db-id=1\n" \ + "db-system-id=6625592122879095702\n" \ + "db-version=\"9.4\"\n" + + #define TEST_MANIFEST_OPTION_ALL \ + "\n" \ + "[backup:option]\n" \ + "option-archive-check=false\n" \ + "option-archive-copy=false\n" \ + "option-checksum-page=false\n" \ + "option-compress=false\n" \ + "option-compress-type=\"none\"\n" \ + "option-hardlink=false\n" \ + "option-online=false\n" + + #define TEST_MANIFEST_OPTION_ARCHIVE_TRUE \ + "\n" \ + "[backup:option]\n" \ + "option-archive-check=true\n" \ + "option-archive-copy=true\n" + + #define TEST_MANIFEST_TARGET \ + "\n" \ + "[backup:target]\n" \ + "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n" + + #define TEST_MANIFEST_DB \ + "\n" \ + "[db]\n" \ + "postgres={\"db-id\":12173,\"db-last-system-id\":12168}\n" + + #define TEST_MANIFEST_FILE \ + "\n" \ + "[target:file]\n" \ + "pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"master\":true" \ + ",\"size\":4,\"timestamp\":1565282114}\n" + + #define TEST_MANIFEST_FILE_DEFAULT \ + "\n" \ + "[target:file:default]\n" \ + "group=\"group1\"\n" \ + "master=false\n" \ + "mode=\"0600\"\n" \ + "user=\"user1\"\n" + + #define TEST_MANIFEST_LINK \ + "\n" \ + "[target:link]\n" \ + "pg_data/pg_stat={\"destination\":\"../pg_stat\"}\n" + + #define TEST_MANIFEST_LINK_DEFAULT \ + "\n" \ + "[target:link:default]\n" \ + "group=\"group1\"\n" \ + "user=false\n" + + #define TEST_MANIFEST_PATH \ + "\n" \ + "[target:path]\n" \ + "pg_data={\"user\":\"user1\"}\n" \ + + #define TEST_MANIFEST_PATH_DEFAULT \ + "\n" \ + "[target:path:default]\n" \ + "group=false\n" \ + "mode=\"0700\"\n" \ + "user=\"user1\"\n" + + // ***************************************************************************************************************************** + if (testBegin("verifyManifestFile()")) + { + // Load Parameters + StringList *argList = strLstDup(argListBase); + harnessCfgLoad(cfgCmdVerify, argList); + + const Buffer *contentLoad = harnessInfoChecksumZ + ( + TEST_MANIFEST_HEADER + TEST_MANIFEST_DB_92 + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + TEST_MANIFEST_FILE + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT + ); + + Manifest *manifest = NULL; + String *backupLabel = strNew("20181119-152138F"); + String *manifestFile = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabel)); + String *manifestFileCopy = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFile)); + unsigned int jobErrorTotal = 0; + VerifyBackupResult backupResult = {.backupLabel = strDup(backupLabel)}; + + InfoArchive *archiveInfo = NULL; + TEST_ASSIGN(archiveInfo, infoArchiveNewLoad(ioBufferReadNew(archiveInfoBase)), "archive.info"); + InfoPg *infoPg = infoArchivePg(archiveInfo); + + //-------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("manifest.copy exists, no manifest main, manifest db version not in history, not current db"); + + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write manifest db section mismatch"); + + backupResult.status = backupValid; + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest"); + TEST_RESULT_PTR(manifest, NULL, "manifest not set - pg version mismatch"); + TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid"); + harnessLogResult( + strZ(strNewFmt( + "P00 WARN: unable to open missing file '%s/%s/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT "' for read\n" + "P00 ERROR: [028]: '%s' may not be recoverable - PG data (id 1, version 9.2, system-id 6625592122879095702) is not " + "in the backup.info history, skipping", + testPath(), strZ(backupStanzaPath), strZ(backupLabel), strZ(backupLabel)))); + + //-------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("rerun test with db-system-id invalid and no main"); + + contentLoad = harnessInfoChecksumZ + ( + TEST_MANIFEST_HEADER + "\n" + "[backup:db]\n" + "db-catalog-version=201409291\n" + "db-control-version=942\n" + "db-id=1\n" + "db-system-id=0\n" + "db-version=\"9.4\"\n" + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + TEST_MANIFEST_FILE + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT + ); + + TEST_RESULT_VOID(storageRemoveP(storageTest, manifestFile), "remove main manifest"); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write manifest copy invalid system-id"); + + backupResult.status = backupValid; + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest"); + TEST_RESULT_PTR(manifest, NULL, "manifest not set - pg system-id mismatch"); + TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid"); + harnessLogResult( + strZ(strNewFmt( + "P00 WARN: unable to open missing file '%s/%s/%s/" BACKUP_MANIFEST_FILE "' for read\n" + "P00 WARN: %s/backup.manifest is missing or unusable, using copy\n" + "P00 ERROR: [028]: '%s' may not be recoverable - PG data (id 1, version 9.4, system-id 0) is not " + "in the backup.info history, skipping", + testPath(), strZ(backupStanzaPath), strZ(backupLabel), strZ(backupLabel), strZ(backupLabel)))); + + //-------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("rerun copy test with db-id invalid"); + + contentLoad = harnessInfoChecksumZ + ( + TEST_MANIFEST_HEADER + "\n" + "[backup:db]\n" + "db-catalog-version=201409291\n" + "db-control-version=942\n" + "db-id=0\n" + "db-system-id=6625592122879095702\n" + "db-version=\"9.4\"\n" + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + TEST_MANIFEST_FILE + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT + ); + + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write manifest copy invalid db-id"); + + backupResult.status = backupValid; + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest"); + TEST_RESULT_PTR(manifest, NULL, "manifest not set - pg db-id mismatch"); + TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid"); + harnessLogResult( + strZ(strNewFmt( + "P00 WARN: unable to open missing file '%s/%s/%s/" BACKUP_MANIFEST_FILE "' for read\n" + "P00 WARN: %s/backup.manifest is missing or unusable, using copy\n" + "P00 ERROR: [028]: '%s' may not be recoverable - PG data (id 0, version 9.4, system-id 6625592122879095702) is not " + "in the backup.info history, skipping", + testPath(), strZ(backupStanzaPath), strZ(backupLabel), strZ(backupLabel), strZ(backupLabel)))); + + //-------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("missing main manifest, errored copy"); + + backupResult.status = backupValid; + contentLoad = BUFSTRDEF( + "[backrest]\n" + "backrest-checksum=\"BOGUS\"\n" + "backrest-format=5\n" + "backrest-version=\"2.28\"\n"); + + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write invalid manifest copy"); + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest"); + TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid"); + harnessLogResult( + strZ(strNewFmt( + "P00 WARN: unable to open missing file '%s/%s/%s/" BACKUP_MANIFEST_FILE "' for read\n" + "P00 WARN: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' " + STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, testPath(), strZ(backupStanzaPath), strZ(backupLabel), + strZ(backupLabel)))); + + //-------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("current backup true"); + + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write invalid manifest"); + + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, true, infoPg, &jobErrorTotal), "verify manifest"); + TEST_RESULT_PTR(manifest, NULL, "manifest not set"); + TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid"); + harnessLogResult( + strZ(strNewFmt( + "P00 WARN: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' " + STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE "\n" + "P00 WARN: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' " + STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(backupLabel), strZ(backupLabel)))); + + // Write a valid manifest with a manifest copy that is invalid + contentLoad = harnessInfoChecksumZ + ( + TEST_MANIFEST_HEADER + TEST_MANIFEST_DB_94 + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + TEST_MANIFEST_FILE + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT + ); + + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write valid manifest"); + + backupResult.status = backupValid; + TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, true, infoPg, &jobErrorTotal), "verify manifest"); + TEST_RESULT_PTR_NE(manifest, NULL, "manifest set"); + TEST_RESULT_UINT(backupResult.status, backupValid, "manifest usable"); + harnessLogResult(strZ(strNewFmt("P00 WARN: backup '%s' manifest.copy does not match manifest", strZ(backupLabel)))); + } + // ***************************************************************************************************************************** if (testBegin("verifyCreateArchiveIdRange()")) { @@ -378,15 +680,6 @@ testRun(void) unsigned int errTotal = 0; - TEST_RESULT_STR_Z( - verifySetBackupCheckArchive( - strLstNew(), backupInfo, strLstNew(), pgHistory, &errTotal), NULL, "no archives or backups"); - TEST_RESULT_UINT(errTotal, 0, "no error"); - TEST_RESULT_STR_Z( - verifySetBackupCheckArchive( - backupList, backupInfo, archiveIdList, pgHistory, &errTotal), NULL, "no current backup, no missing archive"); - TEST_RESULT_UINT(errTotal, 0, "no error"); - // Add backup to end of list strLstAddZ(backupList, "20181119-153000F"); strLstAddZ(archiveIdList, "12-3"); @@ -408,13 +701,17 @@ testRun(void) //-------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("verifyLogInvalidResult() - missing file"); - TEST_RESULT_UINT(verifyLogInvalidResult(verifyFileMissing, 0, strNew("missingfilename")), 0, "file missing message"); + TEST_RESULT_UINT( + verifyLogInvalidResult(STORAGE_REPO_ARCHIVE_STR, verifyFileMissing, 0, strNew("missingfilename")), + 0, "file missing message"); harnessLogResult("P00 WARN: file missing 'missingfilename'"); //-------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("verifyRender() - missing file, empty invalidList"); List *archiveIdResultList = lstNewP(sizeof(VerifyArchiveResult), .comparator = archiveIdComparator); + List *backupResultList = lstNewP(sizeof(VerifyBackupResult), .comparator = lstComparatorStr); + VerifyArchiveResult archiveIdResult = { .archiveId = strNew("9.6-1"), @@ -431,25 +728,40 @@ testRun(void) lstAdd(archiveIdResult.walRangeList, &walRange); lstAdd(archiveIdResultList, &archiveIdResult); TEST_RESULT_STR_Z( - verifyRender(archiveIdResultList), + verifyRender(archiveIdResultList, backupResultList), "Results:\n" " archiveId: 9.6-1, total WAL checked: 1, total valid WAL: 0\n" - " missing: 0, checksum invalid: 0, size invalid: 0, other: 0", "no invalid file list"); + " missing: 0, checksum invalid: 0, size invalid: 0, other: 0\n" + " backup: none found", "archive: no invalid file list"); VerifyInvalidFile invalidFile = { .fileName = strNew("file"), .reason = verifyFileMissing, }; - lstAdd(walRange.invalidFileList, &invalidFile); + + VerifyBackupResult backupResult = + { + .backupLabel = strNew("test-backup-label"), + .status = backupInvalid, + .totalFileVerify = 1, + .invalidFileList = lstNewP(sizeof(VerifyInvalidFile), .comparator = lstComparatorStr), + }; + lstAdd(backupResult.invalidFileList, &invalidFile); + lstAdd(backupResultList, &backupResult); + TEST_RESULT_STR_Z( - verifyRender(archiveIdResultList), + verifyRender(archiveIdResultList, backupResultList), "Results:\n" " archiveId: 9.6-1, total WAL checked: 1, total valid WAL: 0\n" - " missing: 1, checksum invalid: 0, size invalid: 0, other: 0", "file missing"); + " missing: 1, checksum invalid: 0, size invalid: 0, other: 0\n" + " backup: test-backup-label, status: invalid, total files checked: 1, total valid files: 0\n" + " missing: 1, checksum invalid: 0, size invalid: 0, other: 0", "archive file missing, backup file missing"); + + //-------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("verifyAddInvalidWalFile() - file missing (coverage test)"); - // Coverage test TEST_RESULT_VOID( verifyAddInvalidWalFile(archiveIdResult.walRangeList, verifyFileMissing, strNew("test"), strNew("3")), "coverage test"); } @@ -461,20 +773,6 @@ testRun(void) StringList *argList = strLstDup(argListBase); harnessCfgLoad(cfgCmdVerify, argList); - //-------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("neither backup nor archive info files exist"); - - TEST_ERROR(cmdVerify(), RuntimeError, "2 fatal errors encountered, see log for details"); - harnessLogResult( - strZ(strNewFmt( - "P00 WARN: unable to open missing file '%s/%s/backup.info' for read\n" - "P00 WARN: unable to open missing file '%s/%s/backup.info.copy' for read\n" - "P00 ERROR: [029]: No usable backup.info file\n" - "P00 WARN: unable to open missing file '%s/%s/archive.info' for read\n" - "P00 WARN: unable to open missing file '%s/%s/archive.info.copy' for read\n" - "P00 ERROR: [029]: No usable archive.info file", testPath(), strZ(backupStanzaPath), testPath(), - strZ(backupStanzaPath), testPath(), strZ(archiveStanzaPath), testPath(), strZ(archiveStanzaPath)))); - //-------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("backup.info invalid checksum, neither backup copy nor archive infos exist"); @@ -575,34 +873,6 @@ testRun(void) "P00 WARN: unable to open missing file '%s/%s/backup.info.copy' for read\n" "P00 ERROR: [029]: No usable backup.info file", testPath(), strZ(backupStanzaPath), testPath(), strZ(backupStanzaPath)))); - - //-------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("backup.info.copy valid, archive.info and copy valid, present but empty backup"); - - TEST_RESULT_VOID( - storagePutP(storageNewWriteP(storageTest, backupInfoFileName), backupInfoMultiHistoryBase), - "write valid backup.info"); - TEST_RESULT_VOID( - storagePutP(storageNewWriteP(storageTest, backupInfoFileNameCopy), backupInfoMultiHistoryBase), - "write valid backup.info.copy"); - TEST_RESULT_VOID( - storagePathCreateP(storageTest, strNewFmt("%s/20200810-171426F", strZ(backupStanzaPath))), - "create empty backup label path"); - TEST_RESULT_VOID(cmdVerify(), "no archives, empty backup label path"); - harnessLogResult("P00 WARN: no archives exist in the repo"); - - //-------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("backup.info.copy valid, archive.info and copy valid, present but empty backup, empty archive"); - - TEST_RESULT_VOID( - storagePathCreateP(storageTest, strNewFmt("%s/9.4-1", strZ(archiveStanzaPath))), - "create empty path for archiveId"); - - TEST_RESULT_VOID(cmdVerify(), "no jobs - empty archive id and backup label paths"); - harnessLogResult( - "P00 WARN: archive path '9.4-1' is empty\n" - "P00 INFO: Results:\n" - " archiveId: 9.4-1, total WAL checked: 0, total valid WAL: 0"); } // ***************************************************************************************************************************** @@ -619,19 +889,12 @@ testRun(void) TEST_RESULT_VOID(storagePutP(storageNewWriteP(storageRepoWrite(), filePathName), BUFSTRDEF("")), "put zero-sized file"); TEST_RESULT_UINT(verifyFile(filePathName, STRDEF(HASH_TYPE_SHA1_ZERO), 0, NULL), verifyOk, "file ok"); - const char *fileContents = "acefile"; - uint64_t fileSize = 7; - const String *checksum = STRDEF("d1cd8a7d11daa26814b93eb604e1d49ab4b43770"); - TEST_RESULT_VOID(storagePutP(storageNewWriteP(storageRepoWrite(), filePathName), BUFSTRZ(fileContents)), "put file"); - TEST_RESULT_UINT(verifyFile(filePathName, checksum, fileSize, NULL), verifyOk, "file size ok"); - TEST_RESULT_UINT(verifyFile(filePathName, checksum, 0, NULL), verifySizeInvalid, "file size invalid"); - TEST_RESULT_UINT( - verifyFile(filePathName, strNew("badchecksum"), fileSize, NULL), verifyChecksumMismatch, "file checksum mismatch"); + TEST_RESULT_UINT(verifyFile(filePathName, fileChecksum, 0, NULL), verifySizeInvalid, "file size invalid"); TEST_RESULT_UINT( verifyFile( - strNewFmt(STORAGE_REPO_ARCHIVE "/missingFile"), checksum, 0, NULL), verifyFileMissing, "file missing"); + strNewFmt(STORAGE_REPO_ARCHIVE "/missingFile"), fileChecksum, 0, NULL), verifyFileMissing, "file missing"); // Create a compressed encrypted repo file filePathName = strNew(STORAGE_REPO_BACKUP "/testfile.gz"); @@ -642,7 +905,7 @@ testRun(void) TEST_RESULT_VOID(storagePutP(write, BUFSTRZ(fileContents)), "write encrypted, compressed file"); TEST_RESULT_UINT( - verifyFile(filePathName, checksum, fileSize, strNew("pass")), verifyOk, "file encrypted compressed ok"); + verifyFile(filePathName, fileChecksum, fileSize, strNew("pass")), verifyOk, "file encrypted compressed ok"); TEST_RESULT_UINT( verifyFile( filePathName, strNew("badchecksum"), fileSize, strNew("pass")), verifyChecksumMismatch, @@ -660,7 +923,7 @@ testRun(void) VariantList *paramList = varLstNew(); varLstAdd(paramList, varNewStr(filePathName)); - varLstAdd(paramList, varNewStr(checksum)); + varLstAdd(paramList, varNewStr(fileChecksum)); varLstAdd(paramList, varNewUInt64(fileSize)); varLstAdd(paramList, varNewStrZ("pass")); @@ -672,7 +935,7 @@ testRun(void) } // ***************************************************************************************************************************** - if (testBegin("cmdVerify(), verifyProcess()")) + if (testBegin("cmdVerify(), verifyProcess() - errors")) { //-------------------------------------------------------------------------------------------------------------------------- StringList *argList = strLstDup(argListBase); @@ -697,18 +960,14 @@ testRun(void) storagePutP(storageNewWriteP(storageTest, backupInfoFileName), harnessInfoChecksumZ( "[db]\n" - "db-catalog-version=201707211\n" - "db-control-version=1100\n" - "db-id=2\n" - "db-system-id=6626363367545678089\n" - "db-version=\"11\"\n" + TEST_BACKUP_DB2_11 "\n" "[db:history]\n" - "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," - "\"db-version\":\"9.4\"}\n" - "2={\"db-catalog-version\":201707211,\"db-control-version\":1100,\"db-system-id\":6626363367545678089," - "\"db-version\":\"11\"}")), - "put backup.info files"); + TEST_BACKUP_DB1_HISTORY + "\n" + TEST_BACKUP_DB2_HISTORY + )), + "put backup.info files - no current backups"); storageCopy(storageNewReadP(storageTest, backupInfoFileName), storageNewWriteP(storageTest, backupInfoFileNameCopy)); //-------------------------------------------------------------------------------------------------------------------------- @@ -743,13 +1002,14 @@ testRun(void) TEST_ERROR(cmdVerify(), RuntimeError, "1 fatal errors encountered, see log for details"); harnessLogResult( - strZ(strNewFmt( + strZ(strNew( "P00 WARN: no backups exist in the repo\n" "P00 ERROR: [028]: duplicate WAL '000000020000000700000FFE' for '11-2' exists, skipping\n" "P00 WARN: path '11-2/0000000200000007' does not contain any valid WAL to be processed\n" "P00 INFO: Results:\n" " archiveId: 11-2, total WAL checked: 2, total valid WAL: 0\n" - " missing: 0, checksum invalid: 0, size invalid: 0, other: 0"))); + " missing: 0, checksum invalid: 0, size invalid: 0, other: 0\n" + " backup: none found"))); harnessLogLevelReset(); @@ -797,14 +1057,16 @@ testRun(void) unsigned int errorTotal = 0; TEST_RESULT_STR_Z( verifyProcess(&errorTotal), - "Results:\n" - " archiveId: 9.4-1, total WAL checked: 0, total valid WAL: 0\n" - " archiveId: 11-2, total WAL checked: 4, total valid WAL: 2\n" - " missing: 0, checksum invalid: 1, size invalid: 1, other: 0", + strZ(strNew( + "Results:\n" + " archiveId: 9.4-1, total WAL checked: 0, total valid WAL: 0\n" + " archiveId: 11-2, total WAL checked: 4, total valid WAL: 2\n" + " missing: 0, checksum invalid: 1, size invalid: 1, other: 0\n" + " backup: none found")), "verifyProcess() results"); TEST_RESULT_UINT(errorTotal, 2, "errors"); harnessLogResult( - strZ(strNewFmt( + strZ(strNew( "P00 WARN: no backups exist in the repo\n" "P00 WARN: archive path '9.4-1' is empty\n" "P00 WARN: path '11-2/0000000100000000' does not contain any valid WAL to be processed\n" @@ -848,14 +1110,14 @@ testRun(void) TEST_ERROR(cmdVerify(), RuntimeError, "2 fatal errors encountered, see log for details"); harnessLogResult( - strZ(strNewFmt( + strZ(strNew( "P01 ERROR: [028]: invalid checksum " "'11-2/0000000200000007/000000020000000700000FFD-a6e1a64f0813352bc2e97f116a1800377e17d2e4.gz'\n" "P01 ERROR: [028]: invalid size " "'11-2/0000000200000007/000000020000000700000FFF-ee161f898c9012dd0c28b3fd1e7140b9cf411306'"))); //-------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("valid info files, unreadable WAL file"); + TEST_TITLE("valid info files - various archive/backup errors"); // Load Parameters - single non-default repo argList = strLstNew(); @@ -874,41 +1136,440 @@ testRun(void) walBuffer), "write WAL - file not readable"); + String *backupLabelPriorNoManifest = strNew("20181119-152800F"); + TEST_RESULT_VOID( + storagePathCreateP(storageTest, strNewFmt("%s/%s", strZ(backupStanzaPath), strZ(backupLabelPriorNoManifest))), + "prior backup path missing manifests"); + + String *backupLabelManifestNoTargetFile = strNew("20181119-152810F"); + const Buffer *contentLoad = harnessInfoChecksumZ + ( + TEST_MANIFEST_HEADER + TEST_MANIFEST_DB_94 + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + ); + + String *manifestFileNoTarget = strNewFmt( + "%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabelManifestNoTargetFile)); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileNoTarget), contentLoad), "write manifest without target files"); + + // Create full backup with files + String *backupLabel = strNew("20181119-152900F"); + + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, strNewFmt("%s/%s/pg_data/PG_VERSION", strZ(backupStanzaPath), + strZ(backupLabel))), BUFSTRDEF("BOGUS")), "put checksum-error backup file"); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, strNewFmt("%s/%s/pg_data/testzero", strZ(backupStanzaPath), + strZ(backupLabel))), BUFSTRDEF("")), "put zero-size backup file"); + + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, strNewFmt("%s/%s/pg_data/testvalid", strZ(backupStanzaPath), + strZ(backupLabel))), BUFSTRZ(fileContents)), "put valid file"); + + contentLoad = harnessInfoChecksumZ + ( + strZ(strNewFmt( + "[backup]\n" + "backup-label=\"%s\"\n" + "backup-timestamp-copy-start=0\n" + "backup-timestamp-start=0\n" + "backup-timestamp-stop=0\n" + "backup-type=\"full\"\n" + "\n" + "[backup:db]\n" + TEST_BACKUP_DB2_11 + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + TEST_MANIFEST_FILE + "pg_data/testvalid={\"checksum\":\"%s\",\"master\":true,\"size\":7,\"timestamp\":1565282114}\n" + "pg_data/testzero={\"repo-size\":20,\"size\":0,\"timestamp\":1601405663}\n" + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT, + strZ(backupLabel), strZ(fileChecksum))) + ); + + // Write manifests for full backup + String *manifestFile = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabel)); + String *manifestFileCopy = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFile)); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write valid manifest"); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), + "write valid manifest copy"); + + // Create a manifest for the dependent that has references + String *backupLabelDependent = strNew("20181119-152900F_20181119-152909D"); + + // Create an unprocessed backup label with a file that will be referenced in this manifest + String *unprocessedBackup = strNew("UNPROCESSEDBACKUP"); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, strNewFmt("%s/%s/pg_data/testother", strZ(backupStanzaPath), + strZ(unprocessedBackup)), .modeFile = 0200), BUFSTRZ(fileContents)), "put unreadable file to unprocessed backup"); + + contentLoad = harnessInfoChecksumZ + ( + strZ(strNewFmt( + "[backup]\n" + "backup-label=\"%s\"\n" + "backup-timestamp-copy-start=0\n" + "backup-timestamp-start=0\n" + "backup-timestamp-stop=0\n" + "backup-type=\"diff\"\n" + "\n" + "[backup:db]\n" + TEST_BACKUP_DB2_11 + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + "\n" + "[target:file]\n" + "pg_data/PG_VERSION=" + "{\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"master\":true,\"reference\":\"%s\",\"size\":4," + "\"timestamp\":1565282114}\n" + "pg_data/testfile={\"checksum\":\"%s\",\"master\":true,\"reference\":\"%s\",\"size\":7,\"timestamp\":1565282114}\n" + "pg_data/testfile2={\"checksum\":\"%s\",\"master\":true,\"size\":7,\"timestamp\":1565282114}\n" + "pg_data/testmissing=" + "{\"checksum\":\"123473f470864e067ee3a22e64b47b0a1c356abc\",\"size\":7,\"timestamp\":1565282114}\n" + "pg_data/testother={\"checksum\":\"%s\",\"master\":true,\"reference\":\"%s\",\"size\":7,\"timestamp\":1565282114}\n" + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT, + strZ(backupLabelDependent), strZ(backupLabel), strZ(fileChecksum), strZ(backupLabel), strZ(fileChecksum), + strZ(fileChecksum), strZ(unprocessedBackup))) + ); + + // Write manifests for dependent backup + manifestFile = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabelDependent)); + manifestFileCopy = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFile)); + + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write manifest to dependent"); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write manifest copy to dependent"); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, strNewFmt("%s/%s/pg_data/testfile2", strZ(backupStanzaPath), + strZ(backupLabelDependent))), BUFSTRZ(fileContents)), "put valid file to dependent"); + + // Create in-progress backup + TEST_RESULT_VOID( + storagePathCreateP(storageTest, strNewFmt("%s/%s", strZ(backupStanzaPath), "20181119-153000F")), + "create empty backup path for newest backup so in-progress"); + // Set log level to capture ranges harnessLogLevelSet(logLevelDetail); - TEST_ERROR(cmdVerify(), RuntimeError, "3 fatal errors encountered, see log for details"); + TEST_ERROR(cmdVerify(), RuntimeError, "7 fatal errors encountered, see log for details"); harnessLogResult( strZ(strNewFmt( - "P00 WARN: no backups exist in the repo\n" "P00 WARN: archive path '9.4-1' is empty\n" "P00 WARN: path '11-2/0000000100000000' does not contain any valid WAL to be processed\n" "P01 ERROR: [028]: invalid checksum " "'11-2/0000000200000007/000000020000000700000FFD-a6e1a64f0813352bc2e97f116a1800377e17d2e4.gz'\n" "P01 ERROR: [028]: invalid size " "'11-2/0000000200000007/000000020000000700000FFF-ee161f898c9012dd0c28b3fd1e7140b9cf411306'\n" - "P01 ERROR: [039]: invalid verify " + "P01 ERROR: [039]: invalid result " "11-2/0000000200000008/000000020000000800000003-656817043007aa2100c44c712bcb456db705dab9: [41] raised from " "local-1 protocol: unable to open file " "'%s/%s/11-2/0000000200000008/000000020000000800000003-656817043007aa2100c44c712bcb456db705dab9' for read: " "[13] Permission denied\n" + "P00 WARN: unable to open missing file '%s/%s/20181119-152800F/backup.manifest' for read\n" + "P00 WARN: unable to open missing file '%s/%s/20181119-152800F/backup.manifest.copy' for read\n" + "P00 WARN: manifest missing for '20181119-152800F' - backup may have expired\n" + "P00 WARN: unable to open missing file '%s/%s/20181119-152810F/backup.manifest.copy' for read\n" + "P00 ERROR: [028]: backup '20181119-152810F' manifest does not contain any target files to verify\n" + "P01 ERROR: [028]: invalid checksum '20181119-152900F/pg_data/PG_VERSION'\n" + "P01 ERROR: [028]: file missing '20181119-152900F_20181119-152909D/pg_data/testmissing'\n" + "P00 WARN: unable to open missing file '%s/%s/20181119-153000F/backup.manifest' for read\n" + "P00 INFO: backup '20181119-153000F' appears to be in progress, skipping\n" + "P01 ERROR: [039]: invalid result UNPROCESSEDBACKUP/pg_data/testother: [41] raised from local-1 protocol: unable " + "to open file '%s/%s/UNPROCESSEDBACKUP/pg_data/testother' for read: [13] Permission denied\n" "P00 DETAIL: archiveId: 11-2, wal start: 000000020000000700000FFD, wal stop: 000000020000000800000000\n" "P00 DETAIL: archiveId: 11-2, wal start: 000000020000000800000002, wal stop: 000000020000000800000003\n" "P00 DETAIL: archiveId: 11-2, wal start: 000000030000000000000000, wal stop: 000000030000000000000001\n" "P00 INFO: Results:\n" " archiveId: 9.4-1, total WAL checked: 0, total valid WAL: 0\n" " archiveId: 11-2, total WAL checked: 8, total valid WAL: 5\n" - " missing: 0, checksum invalid: 1, size invalid: 1, other: 1", - testPath(), strZ(archiveStanzaPath)))); + " missing: 0, checksum invalid: 1, size invalid: 1, other: 1\n" + " backup: 20181119-152800F, status: manifest missing, total files checked: 0, total valid files: 0\n" + " backup: 20181119-152810F, status: invalid, total files checked: 0, total valid files: 0\n" + " backup: 20181119-152900F, status: invalid, total files checked: 3, total valid files: 2\n" + " missing: 0, checksum invalid: 1, size invalid: 0, other: 0\n" + " backup: 20181119-152900F_20181119-152909D, status: invalid, total files checked: 5, " + "total valid files: 2\n" + " missing: 1, checksum invalid: 1, size invalid: 0, other: 1\n" + " backup: 20181119-153000F, status: in-progress, total files checked: 0, total valid files: 0", + testPath(), strZ(archiveStanzaPath), testPath(), strZ(backupStanzaPath), testPath(), strZ(backupStanzaPath), + testPath(), strZ(backupStanzaPath), testPath(), strZ(backupStanzaPath), testPath(), strZ(backupStanzaPath)))); harnessLogLevelReset(); + } + + // ***************************************************************************************************************************** + if (testBegin("cmdVerify()")) + { + // Load Parameters + StringList *argList = strLstDup(argListBase); + harnessCfgLoad(cfgCmdVerify, argList); + + // Backup labels + String *backupLabelFull = strNew("20181119-152900F"); + String *backupLabelDiff = strNew("20181119-152900F_20181119-152909D"); + String *backupLabelFullDb2 = strNew("20201119-163000F"); + + #define TEST_BACKUP_DB1_CURRENT_FULL3_DIFF1 \ + "20181119-152900F_20181119-152909D={" \ + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," \ + "\"backup-archive-start\":\"000000010000000000000006\",\"backup-archive-stop\":\"000000010000000000000007\"," \ + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \ + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \ + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," \ + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \ + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + + #define TEST_BACKUP_DB2_CURRENT_FULL1 \ + "20201119-163000F={" \ + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," \ + "\"backup-archive-start\":\"000000020000000000000001\",\"backup-archive-stop\":\"000000020000000000000001\"," \ + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \ + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \ + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," \ + "\"db-id\":2,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \ + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + + //-------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("prior backup verification incomplete - referenced file checked"); TEST_RESULT_VOID( - storageRemoveP( - storageTest, - strNewFmt("%s/11-2/0000000200000008/000000020000000800000003-656817043007aa2100c44c712bcb456db705dab9", - strZ(archiveStanzaPath))), - "remove unreadable WAL"); + storagePutP(storageNewWriteP(storageTest, archiveInfoFileName), archiveInfoMultiHistoryBase), + "write archive.info"); + storageCopy(storageNewReadP(storageTest, archiveInfoFileName), storageNewWriteP(storageTest, archiveInfoFileNameCopy)); + + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, backupInfoFileName), + harnessInfoChecksumZ( + "[backup:current]\n" + TEST_BACKUP_DB1_CURRENT_FULL3 + TEST_BACKUP_DB1_CURRENT_FULL3_DIFF1 + TEST_BACKUP_DB2_CURRENT_FULL1 + "\n" + "[db]\n" + TEST_BACKUP_DB2_11 + "\n" + "[db:history]\n" + TEST_BACKUP_DB1_HISTORY + "\n" + TEST_BACKUP_DB2_HISTORY + )), + "write backup.info"); + storageCopy(storageNewReadP(storageTest, backupInfoFileName), storageNewWriteP(storageTest, backupInfoFileNameCopy)); + + // Create valid full backup and valid diff backup for DB1 + const Buffer *contentLoad = harnessInfoChecksumZ + ( + TEST_MANIFEST_HEADER + TEST_MANIFEST_DB_94 + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + TEST_MANIFEST_FILE + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT + ); + + // Write manifests for full backup + String *manifestFile = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabelFull)); + String *manifestFileCopy = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFile)); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write valid manifest - full"); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write valid manifest copy - full"); + + contentLoad = harnessInfoChecksumZ + ( + strZ(strNewFmt( + TEST_MANIFEST_HEADER + TEST_MANIFEST_DB_94 + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + "\n" + "[target:file]\n" + "pg_data/PG_VERSION=" + "{\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"master\":true,\"reference\":\"%s\",\"size\":4," + "\"timestamp\":1565282114}\n" + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT, + strZ(backupLabelFull))) + ); + + // Write manifests for diff backup + String *manifestFileDiff = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabelDiff)); + String *manifestFileCopyDiff = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFileDiff)); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileDiff), contentLoad), "write valid manifest - diff"); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileCopyDiff), contentLoad), "write valid manifest copy - diff"); + + // Put the file referenced by both backups into the full backup + String *filePathName = strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION", strZ(backupLabelFull)); + TEST_RESULT_VOID(storagePutP(storageNewWriteP(storageRepoWrite(), filePathName), BUFSTRZ(fileContents)), "put file"); + + TEST_ERROR(cmdVerify(), RuntimeError, "2 fatal errors encountered, see log for details"); + + // The error for the referenced file is logged twice because it is checked again by the second backup since the first backup + // verification had not yet completed before the second backup verification began + harnessLogResult( + strZ(strNewFmt( + "P00 WARN: no archives exist in the repo\n" + "P01 ERROR: [028]: invalid checksum '%s/pg_data/PG_VERSION'\n" + "P01 ERROR: [028]: invalid checksum '%s/pg_data/PG_VERSION'\n" + "P00 INFO: Results:\n" + " archiveId: none found\n" + " backup: %s, status: invalid, total files checked: 1, total valid files: 0\n" + " missing: 0, checksum invalid: 1, size invalid: 0, other: 0\n" + " backup: %s, status: invalid, total files checked: 1, total valid files: 0\n" + " missing: 0, checksum invalid: 1, size invalid: 0, other: 0", + strZ(backupLabelFull), strZ(backupLabelFull), strZ(backupLabelFull), strZ(backupLabelDiff)))); + + //-------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("valid backup, prior backup verification complete - referenced file not checked"); + + // Set process max to 1 and add more files to check so first backup completes before second is checked + strLstAddZ(argList, "--process-max=1"); + harnessCfgLoad(cfgCmdVerify, argList); + + contentLoad = harnessInfoChecksumZ + ( + strZ(strNewFmt( + TEST_MANIFEST_HEADER + TEST_MANIFEST_DB_94 + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + TEST_MANIFEST_FILE + "pg_data/base/1/555_init=" + "{\"checksum\":\"%s\",\"master\":false,\"size\":1,\"timestamp\":1565282114}\n" + "pg_data/base/1/555_init.1={\"master\":false,\"size\":0,\"timestamp\":1565282114}\n" + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT, + strZ(fileChecksum))) + ); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write valid manifest - full"); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write valid manifest copy - full"); + filePathName = strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/base/1/555_init", strZ(backupLabelFull)); + TEST_RESULT_VOID(storagePutP(storageNewWriteP(storageRepoWrite(), filePathName), BUFSTRZ(fileContents)), + "put file - invalid size"); + + contentLoad = harnessInfoChecksumZ + ( + strZ(strNewFmt( + TEST_MANIFEST_HEADER + TEST_MANIFEST_DB_94 + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + "\n" + "[target:file]\n" + "pg_data/PG_VERSION=" + "{\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"master\":true,\"reference\":\"%s\",\"size\":4," + "\"timestamp\":1565282114}\n" + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT, + strZ(backupLabelFull))) + ); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileDiff), contentLoad), "write valid manifest - diff"); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileCopyDiff), contentLoad), "write valid manifest copy - diff"); + + // Create valid full backup and valid diff backup + contentLoad = harnessInfoChecksumZ + ( + strZ(strNewFmt( + TEST_MANIFEST_HEADER + "\n" + "[backup:db]\n" + TEST_BACKUP_DB2_11 + TEST_MANIFEST_OPTION_ALL + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + "\n" + "[target:file]\n" + "pg_data/validfile={\"checksum\":\"%s\",\"master\":true,\"size\":%u,\"timestamp\":1565282114}\n" + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT, + strZ(fileChecksum), (unsigned int)fileSize)) + ); + + manifestFile = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabelFullDb2)); + manifestFileCopy = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFile)); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write valid manifest - full"); + TEST_RESULT_VOID( + storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write valid manifest copy - full"); + filePathName = strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/validfile", strZ(backupLabelFullDb2)); + TEST_RESULT_VOID(storagePutP(storageNewWriteP(storageRepoWrite(), filePathName), BUFSTRZ(fileContents)), "put valid file"); + + // Create WAL file with just header info and small WAL size + Buffer *walBuffer = bufNew((size_t)(1024 * 1024)); + bufUsedSet(walBuffer, bufSize(walBuffer)); + memset(bufPtr(walBuffer), 0, bufSize(walBuffer)); + pgWalTestToBuffer( + (PgWal){.version = PG_VERSION_11, .systemId = 6626363367545678089, .size = 1024 * 1024}, walBuffer); + const char *walBufferSha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer))); + TEST_RESULT_VOID( + storagePutP( + storageNewWriteP( + storageTest, + strNewFmt("%s/11-2/0000000200000000/000000020000000000000001-%s", strZ(archiveStanzaPath), walBufferSha1)), + walBuffer), + "write valid WAL"); + + TEST_ERROR(cmdVerify(), RuntimeError, "3 fatal errors encountered, see log for details"); + + harnessLogResult( + strZ(strNewFmt( + "P01 ERROR: [028]: invalid checksum '%s/pg_data/PG_VERSION'\n" + "P01 ERROR: [028]: invalid size '%s/pg_data/base/1/555_init'\n" + "P01 ERROR: [028]: file missing '%s/pg_data/base/1/555_init.1'\n" + "P00 INFO: Results:\n" + " archiveId: 11-2, total WAL checked: 1, total valid WAL: 1\n" + " missing: 0, checksum invalid: 0, size invalid: 0, other: 0\n" + " backup: %s, status: invalid, total files checked: 3, total valid files: 0\n" + " missing: 1, checksum invalid: 1, size invalid: 1, other: 0\n" + " backup: %s, status: invalid, total files checked: 1, total valid files: 0\n" + " missing: 0, checksum invalid: 1, size invalid: 0, other: 0\n" + " backup: %s, status: valid, total files checked: 1, total valid files: 1\n" + " missing: 0, checksum invalid: 0, size invalid: 0, other: 0", + strZ(backupLabelFull), strZ(backupLabelFull), strZ(backupLabelFull), strZ(backupLabelFull), + strZ(backupLabelDiff), strZ(backupLabelFullDb2)))); } FUNCTION_HARNESS_RESULT_VOID();