diff --git a/doc/xml/release.xml b/doc/xml/release.xml
index 87a125820..29c5a852d 100644
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -48,6 +48,15 @@
<p>Partial multi-repository implementation.</p>
</release-item>
+ <release-item>
+ <release-item-contributor-list>
+ <release-item-contributor id="cynthia.shang"/>
+ <release-item-reviewer id="david.steele"/>
+ </release-item-contributor-list>
+
+ <p>Add backup verification to internal verify command.</p>
+ </release-item>
+
<release-item>
<release-item-contributor-list>
<release-item-reviewer id="cynthia.shang"/>
diff --git a/src/command/verify/verify.c b/src/command/verify/verify.c
index 1f92e6cbb..2f7df9751 100644
--- a/src/command/verify/verify.c
+++ b/src/command/verify/verify.c
@@ -5,8 +5,9 @@ Verify the contents of the repository.
***********************************************************************************************************************************/
#include "build.auto.h"
-#include <unistd.h>
#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
#include "command/archive/common.h"
#include "command/check/common.h"
@@ -36,11 +37,17 @@ Data Types and Structures
#define FUNCTION_LOG_VERIFY_ARCHIVE_RESULT_FORMAT(value, buffer, bufferSize) \
objToLog(&value, "VerifyArchiveResult", buffer, bufferSize)
+#define FUNCTION_LOG_VERIFY_BACKUP_RESULT_TYPE \
+ VerifyBackupResult
+#define FUNCTION_LOG_VERIFY_BACKUP_RESULT_FORMAT(value, buffer, bufferSize) \
+ objToLog(&value, "VerifyBackupResult", buffer, bufferSize)
+
// Structure for verifying repository info files
typedef struct VerifyInfoFile
{
InfoBackup *backup; // Backup.info file contents
InfoArchive *archive; // Archive.info file contents
+ Manifest *manifest; // Manifest file contents
const String *checksum; // File checksum
int errorCode; // Error code else 0 for no error
} VerifyInfoFile;
@@ -70,6 +77,31 @@ typedef struct VerifyInvalidFile
VerifyResult reason; // Reason file is invalid (e.g. incorrect checksum)
} VerifyInvalidFile;
+// Status result of a backup
+typedef enum
+{
+ backupValid, // Default: All files in backup label repo passed verification
+ backupInvalid, // One of more files in backup label repo failed verification
+ backupMissingManifest, // Backup manifest missing (backup may have expired)
+ backupInProgress, // Backup appeared to be in progress (so was skipped)
+} VerifyBackupResultStatus;
+
+typedef struct VerifyBackupResult
+{
+ String *backupLabel; // Label assigned to the backup
+ VerifyBackupResultStatus status; // Final status of the backup
+ bool fileVerifyComplete; // Have all the files of the backup completed verification?
+ unsigned int totalFileManifest; // Total number of backup files in the manifest
+ unsigned int totalFileVerify; // Total number of backup files being verified
+ unsigned int totalFileValid; // Total number of backup files that were verified and valid
+ String *backupPrior; // Prior backup that this backup depends on, if any
+ unsigned int pgId; // PG id will be used to find WAL for the backup in the repo
+ unsigned int pgVersion; // PG version will be used with PG id to find WAL in the repo
+ String *archiveStart; // First WAL segment in the backup
+ String *archiveStop; // Last WAL segment in the backup
+ List *invalidFileList; // List of invalid files found in the backup
+} VerifyBackupResult;
+
// Job data stucture for processing and results collection
typedef struct VerifyJobData
{
@@ -78,14 +110,49 @@ typedef struct VerifyJobData
StringList *walPathList; // WAL path list for a single archive id
StringList *walFileList; // WAL file list for a single WAL path
StringList *backupList; // List of backups to verify
+ Manifest *manifest; // Manifest contents with list of files to verify
+ unsigned int manifestFileIdx; // Index of the file within the manifest file list to process
String *currentBackup; // In progress backup, if any
const InfoPg *pgHistory; // Database history list
- bool backupProcessing; // Are we processing WAL or are we processing backup
+ bool backupProcessing; // Are we processing WAL or are we processing backups
+ const String *manifestCipherPass; // Cipher pass for reading backup manifests
const String *walCipherPass; // Cipher pass for reading WAL files
+ const String *backupCipherPass; // Cipher pass for reading backup files referenced in a manifest
unsigned int jobErrorTotal; // Total errors that occurred during the job execution
List *archiveIdResultList; // Archive results
+ List *backupResultList; // Backup results
} VerifyJobData;
+/***********************************************************************************************************************************
+Helper function to add a file to an invalid file list
+***********************************************************************************************************************************/
+static void
+verifyInvalidFileAdd(List *invalidFileList, VerifyResult reason, const String *fileName)
+{
+ FUNCTION_TEST_BEGIN();
+ FUNCTION_TEST_PARAM(LIST, invalidFileList); // Invalid file list to add the filename to
+ FUNCTION_TEST_PARAM(ENUM, reason); // Reason for invalid file
+ FUNCTION_TEST_PARAM(STRING, fileName); // Name of invalid file
+ FUNCTION_TEST_END();
+
+ ASSERT(invalidFileList != NULL);
+ ASSERT(fileName != NULL);
+
+ MEM_CONTEXT_BEGIN(lstMemContext(invalidFileList))
+ {
+ VerifyInvalidFile invalidFile =
+ {
+ .fileName = strDup(fileName),
+ .reason = reason,
+ };
+
+ lstAdd(invalidFileList, &invalidFile);
+ }
+ MEM_CONTEXT_END();
+
+ FUNCTION_TEST_RETURN_VOID();
+}
+
/***********************************************************************************************************************************
Load a file into memory
***********************************************************************************************************************************/
@@ -142,8 +209,10 @@ verifyInfoFile(const String *pathFileName, bool keepFile, const String *cipherPa
{
if (strBeginsWith(pathFileName, INFO_BACKUP_PATH_FILE_STR))
result.backup = infoBackupMove(infoBackupNewLoad(infoRead), memContextPrior());
- else
+ else if (strBeginsWith(pathFileName, INFO_ARCHIVE_PATH_FILE_STR))
result.archive = infoArchiveMove(infoArchiveNewLoad(infoRead), memContextPrior());
+ else
+ result.manifest = manifestMove(manifestNewLoad(infoRead), memContextPrior());
}
else
ioReadDrain(infoRead);
@@ -277,6 +346,136 @@ verifyBackupInfoFile(void)
FUNCTION_LOG_RETURN(INFO_BACKUP, result);
}
+/***********************************************************************************************************************************
+Get the manifest file
+***********************************************************************************************************************************/
+static Manifest *
+verifyManifestFile(
+ VerifyBackupResult *backupResult, const String *cipherPass, bool currentBackup, const InfoPg *pgHistory,
+ unsigned int *jobErrorTotal)
+{
+ FUNCTION_LOG_BEGIN(logLevelDebug);
+ FUNCTION_TEST_PARAM_P(VERIFY_BACKUP_RESULT, backupResult); // The result set for the backup being processed
+ FUNCTION_TEST_PARAM(STRING, cipherPass); // Passphrase to access the manifest file
+ FUNCTION_LOG_PARAM(BOOL, currentBackup); // Is this possibly a backup currently in progress?
+ FUNCTION_TEST_PARAM(INFO_PG, pgHistory); // Database history
+ FUNCTION_TEST_PARAM_P(UINT, jobErrorTotal); // Pointer to the overall job error total
+ FUNCTION_LOG_END();
+
+ Manifest *result = NULL;
+
+ MEM_CONTEXT_TEMP_BEGIN()
+ {
+ String *fileName = strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(backupResult->backupLabel));
+
+ // Get the main manifest file
+ VerifyInfoFile verifyManifestInfo = verifyInfoFile(fileName, true, cipherPass);
+
+ // If the main file did not error, then report on the copy's status and check checksums
+ if (verifyManifestInfo.errorCode == 0)
+ {
+ result = verifyManifestInfo.manifest;
+
+ // The current in-progress backup is only notional until the main file is checked because the backup may have
+ // completed by the time the main manifest is checked here. So having a main manifest file means this backup is not
+ // (or is no longer) the currentBackup.
+ currentBackup = false;
+
+ // Attempt to load the copy and report on it's status but don't keep it in memory
+ VerifyInfoFile verifyManifestInfoCopy = verifyInfoFile(
+ strNewFmt("%s%s", strZ(fileName), INFO_COPY_EXT), false, cipherPass);
+
+ // If the copy loaded successfuly, then check the checksums
+ if (verifyManifestInfoCopy.errorCode == 0)
+ {
+ // If the manifest and manifest.copy checksums don't match each other than one (or both) of the files could be
+ // corrupt so log a warning but trust main
+ if (!strEq(verifyManifestInfo.checksum, verifyManifestInfoCopy.checksum))
+ LOG_WARN_FMT("backup '%s' manifest.copy does not match manifest", strZ(backupResult->backupLabel));
+ }
+ }
+ else
+ {
+ // If this might be an in-progress backup and the main manifest is simply missing, it is assumed the backup is an
+ // actual in-progress backup and verification is skipped, otherwise, if the main is not simply missing, or this is not
+ // an in-progress backup then attempt to load the copy.
+ if (!(currentBackup && verifyManifestInfo.errorCode == errorTypeCode(&FileMissingError)))
+ {
+ currentBackup = false;
+
+ VerifyInfoFile verifyManifestInfoCopy = verifyInfoFile(
+ strNewFmt("%s%s", strZ(fileName), INFO_COPY_EXT), true, cipherPass);
+
+ // If loaded successfully, then return the copy as usable
+ if (verifyManifestInfoCopy.errorCode == 0)
+ {
+ LOG_WARN_FMT("%s/backup.manifest is missing or unusable, using copy", strZ(backupResult->backupLabel));
+
+ result = verifyManifestInfoCopy.manifest;
+ }
+ else if (verifyManifestInfo.errorCode == errorTypeCode(&FileMissingError) &&
+ verifyManifestInfoCopy.errorCode == errorTypeCode(&FileMissingError))
+ {
+ backupResult->status = backupMissingManifest;
+
+ LOG_WARN_FMT("manifest missing for '%s' - backup may have expired", strZ(backupResult->backupLabel));
+ }
+ }
+ else
+ {
+ backupResult->status = backupInProgress;
+
+ LOG_INFO_FMT("backup '%s' appears to be in progress, skipping", strZ(backupResult->backupLabel));
+ }
+ }
+
+ // If found a usable manifest then check that the database it was based on is in the history
+ if (result != NULL)
+ {
+ bool found = false;
+ const ManifestData *manData = manifestData(result);
+
+ // Confirm the PG database information from the manifest is in the history list
+ for (unsigned int infoPgIdx = 0; infoPgIdx < infoPgDataTotal(pgHistory); infoPgIdx++)
+ {
+ InfoPgData pgHistoryData = infoPgData(pgHistory, infoPgIdx);
+
+ if (pgHistoryData.id == manData->pgId && pgHistoryData.systemId == manData->pgSystemId &&
+ pgHistoryData.version == manData->pgVersion)
+ {
+ found = true;
+ break;
+ }
+ }
+
+ // If the PG data is not found in the backup.info history, then error and reset the result
+ if (!found)
+ {
+ LOG_ERROR_FMT(
+ errorTypeCode(&FileInvalidError),
+ "'%s' may not be recoverable - PG data (id %u, version %s, system-id %" PRIu64 ") is not in the backup.info"
+ " history, skipping",
+ strZ(backupResult->backupLabel), manData->pgId, strZ(pgVersionToStr(manData->pgVersion)), manData->pgSystemId);
+
+ manifestFree(result);
+ result = NULL;
+ }
+ else
+ manifestMove(result, memContextPrior());
+ }
+
+ // If the result is NULL and the backup status has not yet been set, then the backup is unusable (invalid)
+ if (result == NULL && backupResult->status == backupValid)
+ {
+ backupResult->status = backupInvalid;
+ (*jobErrorTotal)++;
+ }
+ }
+ MEM_CONTEXT_TEMP_END();
+
+ FUNCTION_LOG_RETURN(MANIFEST, result);
+}
+
/***********************************************************************************************************************************
Check the history in the info files
***********************************************************************************************************************************/
@@ -411,7 +610,7 @@ verifyCreateArchiveIdRange(VerifyArchiveResult *archiveIdResult, StringList *wal
{
.start = strDup(walSegment),
.stop = strDup(walSegment),
- .invalidFileList = lstNewP(sizeof(VerifyInvalidFile), .comparator = lstComparatorStr),
+ .invalidFileList = lstNewP(sizeof(VerifyInvalidFile), .comparator = lstComparatorStr),
};
lstAdd(archiveIdResult->walRangeList, &walRangeNew);
@@ -470,7 +669,7 @@ verifyArchive(void *data)
VerifyArchiveResult archiveIdResult =
{
.archiveId = strDup(archiveId),
- .walRangeList = lstNewP(sizeof(VerifyWalRange), .comparator = lstComparatorStr),
+ .walRangeList = lstNewP(sizeof(VerifyWalRange), .comparator = lstComparatorStr),
};
lstAdd(jobData->archiveIdResultList, &archiveIdResult);
@@ -561,8 +760,9 @@ verifyArchive(void *data)
protocolCommandParamAdd(command, VARUINT64(archiveResult->pgWalInfo.size));
protocolCommandParamAdd(command, VARSTR(jobData->walCipherPass));
- // Assign job to result
- result = protocolParallelJobNew(VARSTR(filePathName), command);
+ // Assign job to result, prepending the archiveId to the key for consistency with backup processing
+ result = protocolParallelJobNew(
+ VARSTR(strNewFmt("%s/%s", strZ(archiveResult->archiveId), strZ(filePathName))), command);
// Remove the file to process from the list
strLstRemoveIdx(jobData->walFileList, 0);
@@ -610,6 +810,229 @@ verifyArchive(void *data)
FUNCTION_TEST_RETURN(result);
}
+/***********************************************************************************************************************************
+Verify the job data backups
+***********************************************************************************************************************************/
+static ProtocolParallelJob *
+verifyBackup(void *data)
+{
+ FUNCTION_TEST_BEGIN();
+ FUNCTION_TEST_PARAM_P(VOID, data);
+ FUNCTION_TEST_END();
+
+ ProtocolParallelJob *result = NULL;
+
+ VerifyJobData *jobData = data;
+
+ // Process backup files, if any
+ while (strLstSize(jobData->backupList) > 0)
+ {
+ result = NULL;
+
+ // If result list is empty or the last processed is not equal to the backup being processed, then intialize the backup
+ // data and results
+ if (lstSize(jobData->backupResultList) == 0 ||
+ !strEq(((VerifyBackupResult *)lstGetLast(jobData->backupResultList))->backupLabel, strLstGet(jobData->backupList, 0)))
+ {
+ MEM_CONTEXT_BEGIN(lstMemContext(jobData->backupResultList))
+ {
+ VerifyBackupResult backupResultNew =
+ {
+ .backupLabel = strDup(strLstGet(jobData->backupList, 0)),
+ .invalidFileList = lstNewP(sizeof(VerifyInvalidFile), .comparator = lstComparatorStr),
+ };
+
+ // Add the backup to the result list
+ lstAdd(jobData->backupResultList, &backupResultNew);
+ }
+ MEM_CONTEXT_END();
+
+ // Get the result just added so it can be updated directly
+ VerifyBackupResult *backupResult = lstGetLast(jobData->backupResultList);
+
+ // If currentBackup is set (meaning the newest backup label on disk was not in the db:current section when the
+ // backup.info file was read) and this is the same label, then set inProgessBackup to true, else false.
+ // inProgressBackup may be changed in verifyManifestFile if a main backup.manifest exists since that would indicate the
+ // backup completed during the verify process.
+ bool inProgressBackup = strEq(jobData->currentBackup, backupResult->backupLabel);
+
+ // Get a usable backup manifest file
+ Manifest *manifest = verifyManifestFile(
+ backupResult, jobData->manifestCipherPass, inProgressBackup, jobData->pgHistory, &jobData->jobErrorTotal);
+
+ // If a usable backup.manifest file is not found
+ if (manifest == NULL)
+ {
+ // Remove this backup from the processing list
+ strLstRemoveIdx(jobData->backupList, 0);
+
+ // No files to process so continue to the next backup in the list
+ continue;
+ }
+ // Initialize the backup results and manifest for processing
+ else
+ {
+ // Move the manifest to the jobData for processing
+ jobData->manifest = manifestMove(manifest, jobData->memContext);
+
+ // Initialize the jobData
+ MEM_CONTEXT_BEGIN(jobData->memContext)
+ {
+ // Get the cipher subpass used to decrypt files in the backup and initialize the file list index
+ jobData->backupCipherPass = strDup(manifestCipherSubPass(jobData->manifest));
+ jobData->manifestFileIdx = 0;
+ }
+ MEM_CONTEXT_END();
+
+ const ManifestData *manData = manifestData(jobData->manifest);
+
+ MEM_CONTEXT_BEGIN(lstMemContext(jobData->backupResultList))
+ {
+ backupResult->totalFileManifest = manifestFileTotal(jobData->manifest);
+ backupResult->backupPrior = strDup(manData->backupLabelPrior);
+ backupResult->pgId = manData->pgId;
+ backupResult->pgVersion = manData->pgVersion;
+ backupResult->archiveStart = strDup(manData->archiveStart);
+ backupResult->archiveStop = strDup(manData->archiveStop);
+ }
+ MEM_CONTEXT_END();
+ }
+ }
+
+ VerifyBackupResult *backupResult = lstGetLast(jobData->backupResultList);
+
+ // Process any files in the manifest
+ if (jobData->manifestFileIdx < manifestFileTotal(jobData->manifest))
+ {
+ do
+ {
+ const ManifestFile *fileData = manifestFile(jobData->manifest, jobData->manifestFileIdx);
+
+ String *filePathName = NULL;
+
+ // Track the files verified in order to determine when the processing of the backup is complete
+ backupResult->totalFileVerify++;
+
+ // Check if the file is referenced in a prior backup
+ if (fileData->reference != NULL)
+ {
+ // If the prior backup is not in the result list, then that backup was never processed (likely due to the --set
+ // option) so verify the file
+ unsigned int backupPriorIdx = lstFindIdx(jobData->backupResultList, &fileData->reference);
+
+ if (backupPriorIdx == LIST_NOT_FOUND)
+ {
+ filePathName = strNewFmt(
+ STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name),
+ strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
+ }
+ // Else the backup this file references has a result so check the processing state for the referenced backup
+ else
+ {
+ VerifyBackupResult *backupResultPrior = lstGet(jobData->backupResultList, backupPriorIdx);
+
+ // If the verify-state of the backup is not complete then verify the file
+ if (!backupResultPrior->fileVerifyComplete)
+ {
+ filePathName = strNewFmt(
+ STORAGE_REPO_BACKUP "/%s/%s%s", strZ(fileData->reference), strZ(fileData->name),
+ strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
+ }
+ // Else skip verification
+ else
+ {
+ String *priorFile = strNewFmt(
+ "%s/%s%s", strZ(fileData->reference), strZ(fileData->name),
+ strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
+
+ unsigned int backupPriorInvalidIdx = lstFindIdx(backupResultPrior->invalidFileList, &priorFile);
+
+ // If the file is in the invalid file list of the prior backup where it is referenced then add the file
+ // as invalid to this backup result and set the backup result status; since already logged an error on
+ // this file, don't log again
+ if (backupPriorInvalidIdx != LIST_NOT_FOUND)
+ {
+ VerifyInvalidFile *invalidFile = lstGet(
+ backupResultPrior->invalidFileList, backupPriorInvalidIdx);
+ verifyInvalidFileAdd(backupResult->invalidFileList, invalidFile->reason, invalidFile->fileName);
+ backupResult->status = backupInvalid;
+ }
+ // Else the file in the prior backup was valid so increment the total valid files for this backup
+ else
+ {
+ backupResult->totalFileValid++;
+ }
+ }
+ }
+ }
+ // Else file is not referenced in a prior backup
+ else
+ {
+ filePathName = strNewFmt(
+ STORAGE_REPO_BACKUP "/%s/%s%s", strZ(backupResult->backupLabel), strZ(fileData->name),
+ strZ(compressExtStr((manifestData(jobData->manifest))->backupOptionCompressType)));
+ }
+
+ // If constructed file name is not null then send it off for processing
+ if (filePathName != NULL)
+ {
+ // Set up the job
+ ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE_STR);
+ protocolCommandParamAdd(command, VARSTR(filePathName));
+
+ // If the checksum is not present in the manifest, it will be calculated by manifest load
+ protocolCommandParamAdd(command, VARSTRZ(fileData->checksumSha1));
+ protocolCommandParamAdd(command, VARUINT64(fileData->size));
+ protocolCommandParamAdd(command, VARSTR(jobData->backupCipherPass));
+
+ // Assign job to result (prepend backup label being processed to the key since some files are in a prior backup)
+ result = protocolParallelJobNew(
+ VARSTR(strNewFmt("%s/%s", strZ(backupResult->backupLabel), strZ(filePathName))), command);
+ }
+
+ // Increment the index to point to the next file
+ jobData->manifestFileIdx++;
+
+ // If this was the last file to process for this backup, then free the manifest and remove this backup from the
+ // processing list
+ if (jobData->manifestFileIdx == backupResult->totalFileManifest)
+ {
+ manifestFree(jobData->manifest);
+ jobData->manifest = NULL;
+ strLstRemoveIdx(jobData->backupList, 0);
+ }
+
+ // If a job was found to be processed then break out to process it
+ if (result != NULL)
+ break;
+ }
+ while (jobData->manifestFileIdx < backupResult->totalFileManifest);
+ }
+ else
+ {
+ // Nothing to process so report an error, free the manifest, set the status, and remove the backup from processing list
+ LOG_ERROR_FMT(
+ errorTypeCode(&FileInvalidError), "backup '%s' manifest does not contain any target files to verify",
+ strZ(backupResult->backupLabel));
+
+ jobData->jobErrorTotal++;
+
+ manifestFree(jobData->manifest);
+ jobData->manifest = NULL;
+
+ backupResult->status = backupInvalid;
+
+ strLstRemoveIdx(jobData->backupList, 0);
+ }
+
+ // If a job was found to be processed then break out to process it
+ if (result != NULL)
+ break;
+ }
+
+ FUNCTION_TEST_RETURN(result);
+}
+
/***********************************************************************************************************************************
Process the job data
***********************************************************************************************************************************/
@@ -626,7 +1049,6 @@ verifyJobCallback(void *data, unsigned int clientIdx)
// Initialize the result
ProtocolParallelJob *result = NULL;
- // Get a new job if there are any left
MEM_CONTEXT_TEMP_BEGIN()
{
VerifyJobData *jobData = data;
@@ -634,8 +1056,17 @@ verifyJobCallback(void *data, unsigned int clientIdx)
if (!jobData->backupProcessing)
{
result = protocolParallelJobMove(verifyArchive(data), memContextPrior());
+
+ // Set the backupProcessing flag if the archive processing is finished so backup processing can begin immediately after
jobData->backupProcessing = strLstSize(jobData->archiveIdList) == 0;
}
+
+ if (jobData->backupProcessing)
+ {
+ // Only begin backup verification if the last archive result was processed
+ if (result == NULL)
+ result = protocolParallelJobMove(verifyBackup(data), memContextPrior());
+ }
}
MEM_CONTEXT_TEMP_END();
@@ -661,7 +1092,7 @@ verifyErrorMsg(VerifyResult verifyResult)
else if (verifyResult == verifySizeInvalid)
result = strCatZ(result, "invalid size");
else
- result = strCatZ(result, "invalid verify");
+ result = strCatZ(result, "invalid result");
FUNCTION_TEST_RETURN(result);
}
@@ -670,19 +1101,21 @@ verifyErrorMsg(VerifyResult verifyResult)
Helper function to output a log message based on job result that is not verifyOk and return an error count
***********************************************************************************************************************************/
static unsigned int
-verifyLogInvalidResult(VerifyResult verifyResult, unsigned int processId, String *filePathName)
+verifyLogInvalidResult(const String *fileType, VerifyResult verifyResult, unsigned int processId, const String *filePathName)
{
FUNCTION_TEST_BEGIN();
+ FUNCTION_TEST_PARAM(STRING, fileType); // Indicates archive or backup file
FUNCTION_TEST_PARAM(ENUM, verifyResult); // Result code from the verifyFile() function
FUNCTION_TEST_PARAM(UINT, processId); // Process Id reporting the result
FUNCTION_TEST_PARAM(STRING, filePathName); // File for which results are being reported
FUNCTION_TEST_END();
+ ASSERT(fileType != NULL);
ASSERT(filePathName != NULL);
// Log a warning because the WAL may have gone missing if expire came through and removed it
// legitimately so it is not necessarily an error so the jobErrorTotal should not be incremented
- if (verifyResult == verifyFileMissing)
+ if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR) && verifyResult == verifyFileMissing)
{
LOG_WARN_PID_FMT(processId, "%s '%s'", strZ(verifyErrorMsg(verifyResult)), strZ(filePathName));
FUNCTION_TEST_RETURN(0);
@@ -787,25 +1220,23 @@ verifyAddInvalidWalFile(List *walRangeList, VerifyResult fileResult, String *fil
ASSERT(fileName != NULL);
ASSERT(walSegment != NULL);
- for (unsigned int walIdx = 0; walIdx < lstSize(walRangeList); walIdx++)
+ MEM_CONTEXT_TEMP_BEGIN()
{
- VerifyWalRange *walRange = lstGet(walRangeList, walIdx);
-
- // If the WAL segment is less/equal to the stop file then it falls in this range since ranges are sorted by stop file in
- // ascending order, therefore first one found is the range
- if (strCmp(walRange->stop, walSegment) >= 0)
+ for (unsigned int walIdx = 0; walIdx < lstSize(walRangeList); walIdx++)
{
- VerifyInvalidFile invalidFile =
- {
- .fileName = strDup(fileName),
- .reason = fileResult,
- };
+ VerifyWalRange *walRange = lstGet(walRangeList, walIdx);
- // Add the file to the range where it was found and exit the loop
- lstAdd(walRange->invalidFileList, &invalidFile);
- break;
+ // If the WAL segment is less/equal to the stop file then it falls in this range since ranges are sorted by stop file in
+ // ascending order, therefore first one found is the range
+ if (strCmp(walRange->stop, walSegment) >= 0)
+ {
+ // Add the file to the range where it was found and exit the loop
+ verifyInvalidFileAdd(walRange->invalidFileList, fileResult, fileName);
+ break;
+ }
}
}
+ MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_VOID();
}
@@ -814,43 +1245,122 @@ verifyAddInvalidWalFile(List *walRangeList, VerifyResult fileResult, String *fil
Render the results of the verify command
***********************************************************************************************************************************/
static String *
-verifyRender(List *archiveIdResultList)
+verifyRender(List *archiveIdResultList, List *backupResultList)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(LIST, archiveIdResultList); // Result list for all archive Ids in the repo
+ FUNCTION_TEST_PARAM(LIST, backupResultList); // Result list for all backups in the repo
FUNCTION_TEST_END();
ASSERT(archiveIdResultList != NULL);
+ ASSERT(backupResultList != NULL);
- String *result = strNew("Results:\n");
+ String *result = strNew("Results:");
- for (unsigned int archiveIdx = 0; archiveIdx < lstSize(archiveIdResultList); archiveIdx++)
+ // Render archive results
+ if (lstSize(archiveIdResultList) == 0)
+ strCatZ(result, "\n archiveId: none found");
+ else
{
- VerifyArchiveResult *archiveIdResult = lstGet(archiveIdResultList, archiveIdx);
- strCatFmt(
- result, "%s archiveId: %s, total WAL checked: %u, total valid WAL: %u", (archiveIdx > 0 ? "\n" : ""),
- strZ(archiveIdResult->archiveId), archiveIdResult->totalWalFile, archiveIdResult->totalValidWal);
-
- if (archiveIdResult->totalWalFile > 0)
+ for (unsigned int archiveIdx = 0; archiveIdx < lstSize(archiveIdResultList); archiveIdx++)
{
- unsigned int errMissing = 0;
- unsigned int errChecksum = 0;
- unsigned int errSize = 0;
- unsigned int errOther = 0;
+ VerifyArchiveResult *archiveIdResult = lstGet(archiveIdResultList, archiveIdx);
+ strCatFmt(
+ result, "\n archiveId: %s, total WAL checked: %u, total valid WAL: %u", strZ(archiveIdResult->archiveId),
+ archiveIdResult->totalWalFile, archiveIdResult->totalValidWal);
- for (unsigned int walIdx = 0; walIdx < lstSize(archiveIdResult->walRangeList); walIdx++)
+ if (archiveIdResult->totalWalFile > 0)
{
- VerifyWalRange *walRange = lstGet(archiveIdResult->walRangeList, walIdx);
+ unsigned int errMissing = 0;
+ unsigned int errChecksum = 0;
+ unsigned int errSize = 0;
+ unsigned int errOther = 0;
- LOG_DETAIL_FMT(
- "archiveId: %s, wal start: %s, wal stop: %s", strZ(archiveIdResult->archiveId), strZ(walRange->start),
- strZ(walRange->stop));
-
- unsigned int invalidIdx = 0;
-
- while (invalidIdx < lstSize(walRange->invalidFileList))
+ for (unsigned int walIdx = 0; walIdx < lstSize(archiveIdResult->walRangeList); walIdx++)
{
- VerifyInvalidFile *invalidFile = lstGet(walRange->invalidFileList, invalidIdx);
+ VerifyWalRange *walRange = lstGet(archiveIdResult->walRangeList, walIdx);
+
+ LOG_DETAIL_FMT(
+ "archiveId: %s, wal start: %s, wal stop: %s", strZ(archiveIdResult->archiveId), strZ(walRange->start),
+ strZ(walRange->stop));
+
+ unsigned int invalidIdx = 0;
+
+ while (invalidIdx < lstSize(walRange->invalidFileList))
+ {
+ VerifyInvalidFile *invalidFile = lstGet(walRange->invalidFileList, invalidIdx);
+
+ if (invalidFile->reason == verifyFileMissing)
+ errMissing++;
+ else if (invalidFile->reason == verifyChecksumMismatch)
+ errChecksum++;
+ else if (invalidFile->reason == verifySizeInvalid)
+ errSize++;
+ else
+ errOther++;
+
+ invalidIdx++;
+ }
+ }
+
+ strCatFmt(
+ result, "\n missing: %u, checksum invalid: %u, size invalid: %u, other: %u", errMissing, errChecksum,
+ errSize, errOther);
+ }
+ }
+ }
+
+ // Render backup results
+ if (lstSize(backupResultList) == 0)
+ strCatZ(result, "\n backup: none found");
+ else
+ {
+ for (unsigned int backupIdx = 0; backupIdx < lstSize(backupResultList); backupIdx++)
+ {
+ VerifyBackupResult *backupResult = lstGet(backupResultList, backupIdx);
+ String *status = NULL;
+
+ switch (backupResult->status)
+ {
+ case backupValid:
+ {
+ status = strNew("valid");
+ break;
+ }
+
+ case backupInvalid:
+ {
+ status = strNew("invalid");
+ break;
+ }
+
+ case backupMissingManifest:
+ {
+ status = strNew("manifest missing");
+ break;
+ }
+
+ case backupInProgress:
+ {
+ status = strNew("in-progress");
+ break;
+ }
+ }
+
+ strCatFmt(
+ result, "\n backup: %s, status: %s, total files checked: %u, total valid files: %u",
+ strZ(backupResult->backupLabel), strZ(status), backupResult->totalFileVerify, backupResult->totalFileValid);
+
+ if (backupResult->totalFileVerify > 0)
+ {
+ unsigned int errMissing = 0;
+ unsigned int errChecksum = 0;
+ unsigned int errSize = 0;
+ unsigned int errOther = 0;
+
+ for (unsigned int invalidIdx = 0; invalidIdx < lstSize(backupResult->invalidFileList); invalidIdx++)
+ {
+ VerifyInvalidFile *invalidFile = lstGet(backupResult->invalidFileList, invalidIdx);
if (invalidFile->reason == verifyFileMissing)
errMissing++;
@@ -860,15 +1370,12 @@ verifyRender(List *archiveIdResultList)
errSize++;
else
errOther++;
-
- invalidIdx++;
}
- }
- strCatFmt(
- result,
- "\n missing: %u, checksum invalid: %u, size invalid: %u, other: %u",
- errMissing, errChecksum, errSize, errOther);
+ strCatFmt(
+ result, "\n missing: %u, checksum invalid: %u, size invalid: %u, other: %u", errMissing, errChecksum,
+ errSize, errOther);
+ }
}
}
@@ -940,18 +1447,20 @@ verifyProcess(unsigned int *errorTotal)
.walPathList = NULL,
.walFileList = strLstNew(),
.pgHistory = infoArchivePg(archiveInfo),
+ .manifestCipherPass = infoPgCipherPass(infoBackupPg(backupInfo)),
.walCipherPass = infoPgCipherPass(infoArchivePg(archiveInfo)),
- .archiveIdResultList = lstNewP(sizeof(VerifyArchiveResult), .comparator = archiveIdComparator),
+ .archiveIdResultList = lstNewP(sizeof(VerifyArchiveResult), .comparator = archiveIdComparator),
+ .backupResultList = lstNewP(sizeof(VerifyBackupResult), .comparator = lstComparatorStr),
};
- // Get a list of backups in the repo
+ // Get a list of backups in the repo sorted ascending
jobData.backupList = strLstSort(
storageListP(
storage, STORAGE_REPO_BACKUP_STR,
.expression = backupRegExpP(.full = true, .differential = true, .incremental = true)),
sortOrderAsc);
- // Get a list of archive Ids in the repo (e.g. 9.4-1, 10-2, etc) sorted by the db-id (number after the dash)
+ // Get a list of archive Ids in the repo (e.g. 9.4-1, 10-2, etc) sorted ascending by the db-id (number after the dash)
jobData.archiveIdList = strLstSort(
strLstComparatorSet(
storageListP(storage, STORAGE_REPO_ARCHIVE_STR, .expression = STRDEF(REGEX_ARCHIVE_DIR_DB_VERSION)),
@@ -966,6 +1475,10 @@ verifyProcess(unsigned int *errorTotal)
if (strLstSize(jobData.archiveIdList) == 0 || strLstSize(jobData.backupList) == 0)
LOG_WARN_FMT("no %s exist in the repo", strLstSize(jobData.archiveIdList) == 0 ? "archives" : "backups");
+ // If there are no archives to process, then set the processing flag to skip to processing the backups
+ if (strLstSize(jobData.archiveIdList) == 0)
+ jobData.backupProcessing = true;
+
// Set current backup if there is one and verify the archive history on disk is in the database history
jobData.currentBackup = verifySetBackupCheckArchive(
jobData.backupList, backupInfo, jobData.archiveIdList, jobData.pgHistory, &jobData.jobErrorTotal);
@@ -989,33 +1502,68 @@ verifyProcess(unsigned int *errorTotal)
ProtocolParallelJob *job = protocolParallelResult(parallelExec);
unsigned int processId = protocolParallelJobProcessId(job);
StringList *filePathLst = strLstNewSplit(varStr(protocolParallelJobKey(job)), FSLASH_STR);
+
+ // Remove the result and file type identifier and recreate the path file name
+ const String *resultId = strLstGet(filePathLst, 0);
+ strLstRemoveIdx(filePathLst, 0);
+ const String *fileType = strLstGet(filePathLst, 0);
strLstRemoveIdx(filePathLst, 0);
String *filePathName = strLstJoin(filePathLst, "/");
+ // Initialize the result sets
VerifyArchiveResult *archiveIdResult = NULL;
+ VerifyBackupResult *backupResult = NULL;
- // Find the archiveId in the list - assert if not found since this should never happen
- String *archiveId = strLstGet(filePathLst, 0);
- unsigned int index = lstFindIdx(jobData.archiveIdResultList, &archiveId);
- ASSERT(index != LIST_NOT_FOUND);
+ // Get archiveId result data
+ if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR))
+ {
+ // Find the archiveId in the list - assert if not found since this should never happen
+ unsigned int index = lstFindIdx(jobData.archiveIdResultList, &resultId);
+ ASSERT(index != LIST_NOT_FOUND);
- archiveIdResult = lstGet(jobData.archiveIdResultList, index);
+ archiveIdResult = lstGet(jobData.archiveIdResultList, index);
+ }
+ // Else get the backup result data
+ else
+ {
+ unsigned int index = lstFindIdx(jobData.backupResultList, &resultId);
+ ASSERT(index != LIST_NOT_FOUND);
+
+ backupResult = lstGet(jobData.backupResultList, index);
+ }
// The job was successful
if (protocolParallelJobErrorCode(job) == 0)
{
const VerifyResult verifyResult = (VerifyResult)varUIntForce(protocolParallelJobResult(job));
- if (verifyResult == verifyOk)
- archiveIdResult->totalValidWal++;
+ // Update the result set for the type of file being processed
+ if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR))
+ {
+ if (verifyResult == verifyOk)
+ archiveIdResult->totalValidWal++;
+ else
+ {
+ jobData.jobErrorTotal += verifyLogInvalidResult(
+ fileType, verifyResult, processId, filePathName);
+
+ // Add invalid file to the WAL range
+ verifyAddInvalidWalFile(
+ archiveIdResult->walRangeList, verifyResult, filePathName,
+ strSubN(strLstGet(filePathLst, strLstSize(filePathLst) - 1), 0, WAL_SEGMENT_NAME_SIZE));
+ }
+ }
else
{
- jobData.jobErrorTotal += verifyLogInvalidResult(verifyResult, processId, filePathName);
-
- // Add invalid file with reason from result of verifyFile to range list
- verifyAddInvalidWalFile(
- archiveIdResult->walRangeList, verifyResult, filePathName,
- strSubN(strLstGet(filePathLst, strLstSize(filePathLst) - 1), 0, WAL_SEGMENT_NAME_SIZE));
+ if (verifyResult == verifyOk)
+ backupResult->totalFileValid++;
+ else
+ {
+ jobData.jobErrorTotal += verifyLogInvalidResult(
+ fileType, verifyResult, processId, filePathName);
+ backupResult->status = backupInvalid;
+ verifyInvalidFileAdd(backupResult->invalidFileList, verifyResult, filePathName);
+ }
}
}
// Else the job errored
@@ -1029,10 +1577,26 @@ verifyProcess(unsigned int *errorTotal)
jobData.jobErrorTotal++;
- // Add invalid file with "OtherError" reason to range list
- verifyAddInvalidWalFile(
- archiveIdResult->walRangeList, verifyOtherError, filePathName,
- strSubN(strLstGet(filePathLst, strLstSize(filePathLst) - 1), 0, WAL_SEGMENT_NAME_SIZE));
+ // Add invalid file with "OtherError" reason to invalid file list
+ if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR))
+ {
+ // Add invalid file to the WAL range
+ verifyAddInvalidWalFile(
+ archiveIdResult->walRangeList, verifyOtherError, filePathName,
+ strSubN(strLstGet(filePathLst, strLstSize(filePathLst) - 1), 0, WAL_SEGMENT_NAME_SIZE));
+ }
+ else
+ {
+ backupResult->status = backupInvalid;
+ verifyInvalidFileAdd(backupResult->invalidFileList, verifyOtherError, filePathName);
+ }
+ }
+
+ // Set backup verification complete for a backup if all files have run through verification
+ if (strEq(fileType, STORAGE_REPO_BACKUP_STR) &&
+ backupResult->totalFileVerify == backupResult->totalFileManifest)
+ {
+ backupResult->fileVerifyComplete = true;
}
// Free the job
@@ -1044,8 +1608,7 @@ verifyProcess(unsigned int *errorTotal)
// ??? Need to do the final reconciliation - checking backup required WAL against, valid WAL
// Report results
- if (lstSize(jobData.archiveIdResultList) > 0)
- resultStr = verifyRender(jobData.archiveIdResultList);
+ resultStr = verifyRender(jobData.archiveIdResultList, jobData.backupResultList);
}
else
LOG_WARN("no archives or backups exist in the repo");
diff --git a/test/define.yaml b/test/define.yaml
index 32aab2ea1..9809a476b 100644
--- a/test/define.yaml
+++ b/test/define.yaml
@@ -712,7 +712,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: verify
- total: 6
+ total: 8
binReq: true
coverage:
diff --git a/test/src/module/command/verifyTest.c b/test/src/module/command/verifyTest.c
index 7f552a688..e7b273ee0 100644
--- a/test/src/module/command/verifyTest.c
+++ b/test/src/module/command/verifyTest.c
@@ -33,68 +33,88 @@ testRun(void)
strLstAdd(argListBase, strNewFmt("--stanza=%s", strZ(stanza)));
strLstAdd(argListBase, strNewFmt("--repo1-path=%s/repo", testPath()));
+ const char *fileContents = "acefile";
+ uint64_t fileSize = 7;
+ const String *fileChecksum = STRDEF("d1cd8a7d11daa26814b93eb604e1d49ab4b43770");
+
+ #define TEST_BACKUP_DB1_94 \
+ "db-catalog-version=201409291\n" \
+ "db-control-version=942\n" \
+ "db-id=1\n" \
+ "db-system-id=6625592122879095702\n" \
+ "db-version=\"9.4\"\n"
+
+ #define TEST_BACKUP_DB2_11 \
+ "db-catalog-version=201707211\n" \
+ "db-control-version=1100\n" \
+ "db-id=2\n" \
+ "db-system-id=6626363367545678089\n" \
+ "db-version=\"11\"\n"
+
+ #define TEST_BACKUP_DB1_CURRENT_FULL1 \
+ "20181119-152138F={" \
+ "\"backrest-format\":5,\"backrest-version\":\"2.28dev\"," \
+ "\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\"," \
+ "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \
+ "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \
+ "\"backup-timestamp-start\":1482182846,\"backup-timestamp-stop\":1482182861,\"backup-type\":\"full\"," \
+ "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \
+ "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
+
+ #define TEST_BACKUP_DB1_CURRENT_FULL2 \
+ "20181119-152800F={" \
+ "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," \
+ "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \
+ "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \
+ "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," \
+ "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \
+ "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
+
+ #define TEST_BACKUP_DB1_CURRENT_FULL3 \
+ "20181119-152900F={" \
+ "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," \
+ "\"backup-archive-start\":\"000000010000000000000004\",\"backup-archive-stop\":\"000000010000000000000004\"," \
+ "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \
+ "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \
+ "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," \
+ "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \
+ "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
+
+ #define TEST_BACKUP_DB1_HISTORY \
+ "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," \
+ "\"db-version\":\"9.4\"}"
+
+ #define TEST_BACKUP_DB2_HISTORY \
+ "2={\"db-catalog-version\":201707211,\"db-control-version\":1100,\"db-system-id\":6626363367545678089," \
+ "\"db-version\":\"11\"}"
+
String *backupInfoContent = strNewFmt(
"[backup:current]\n"
- "20181119-152138F={"
- "\"backrest-format\":5,\"backrest-version\":\"2.28dev\","
- "\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\","
- "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
- "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
- "\"backup-timestamp-start\":1482182846,\"backup-timestamp-stop\":1482182861,\"backup-type\":\"full\","
- "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
- "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
+ TEST_BACKUP_DB1_CURRENT_FULL1
"\n"
"[db]\n"
- "db-catalog-version=201409291\n"
- "db-control-version=942\n"
- "db-id=1\n"
- "db-system-id=6625592122879095702\n"
- "db-version=\"9.4\"\n"
+ TEST_BACKUP_DB1_94
"\n"
"[db:history]\n"
- "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702,"
- "\"db-version\":\"9.4\"}");
+ TEST_BACKUP_DB1_HISTORY
+ );
const Buffer *backupInfoBase = harnessInfoChecksumZ(strZ(backupInfoContent));
String *backupInfoMultiHistoryContent = strNewFmt(
"[backup:current]\n"
- "20181119-152138F={"
- "\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
- "\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\","
- "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
- "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
- "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
- "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
- "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
- "20181119-152800F={"
- "\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
- "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
- "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
- "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
- "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
- "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
- "20181119-152900F={"
- "\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
- "\"backup-archive-start\":\"000000010000000000000004\",\"backup-archive-stop\":\"000000010000000000000004\","
- "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
- "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
- "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
- "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
- "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
+ TEST_BACKUP_DB1_CURRENT_FULL1
+ TEST_BACKUP_DB1_CURRENT_FULL2
+ TEST_BACKUP_DB1_CURRENT_FULL3
"\n"
"[db]\n"
- "db-catalog-version=201707211\n"
- "db-control-version=1100\n"
- "db-id=2\n"
- "db-system-id=6626363367545678089\n"
- "db-version=\"11\"\n"
+ TEST_BACKUP_DB2_11
"\n"
"[db:history]\n"
- "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702,"
- "\"db-version\":\"9.4\"}\n"
- "2={\"db-catalog-version\":201707211,\"db-control-version\":1100,\"db-system-id\":6626363367545678089,"
- "\"db-version\":\"11\"}");
+ TEST_BACKUP_DB1_HISTORY
+ "\n"
+ TEST_BACKUP_DB2_HISTORY
+ );
const Buffer *backupInfoMultiHistoryBase = harnessInfoChecksumZ(strZ(backupInfoMultiHistoryContent));
@@ -121,6 +141,288 @@ testRun(void)
const Buffer *archiveInfoMultiHistoryBase = harnessInfoChecksumZ(strZ(archiveInfoMultiHistoryContent));
+ #define TEST_MANIFEST_HEADER \
+ "[backup]\n" \
+ "backup-label=null\n" \
+ "backup-timestamp-copy-start=0\n" \
+ "backup-timestamp-start=0\n" \
+ "backup-timestamp-stop=0\n" \
+ "backup-type=\"full\"\n"
+
+ #define TEST_MANIFEST_DB_92 \
+ "\n" \
+ "[backup:db]\n" \
+ "db-catalog-version=201204301\n" \
+ "db-control-version=922\n" \
+ "db-id=1\n" \
+ "db-system-id=6625592122879095702\n" \
+ "db-version=\"9.2\"\n"
+
+ #define TEST_MANIFEST_DB_94 \
+ "\n" \
+ "[backup:db]\n" \
+ "db-catalog-version=201409291\n" \
+ "db-control-version=942\n" \
+ "db-id=1\n" \
+ "db-system-id=6625592122879095702\n" \
+ "db-version=\"9.4\"\n"
+
+ #define TEST_MANIFEST_OPTION_ALL \
+ "\n" \
+ "[backup:option]\n" \
+ "option-archive-check=false\n" \
+ "option-archive-copy=false\n" \
+ "option-checksum-page=false\n" \
+ "option-compress=false\n" \
+ "option-compress-type=\"none\"\n" \
+ "option-hardlink=false\n" \
+ "option-online=false\n"
+
+ #define TEST_MANIFEST_OPTION_ARCHIVE_TRUE \
+ "\n" \
+ "[backup:option]\n" \
+ "option-archive-check=true\n" \
+ "option-archive-copy=true\n"
+
+ #define TEST_MANIFEST_TARGET \
+ "\n" \
+ "[backup:target]\n" \
+ "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n"
+
+ #define TEST_MANIFEST_DB \
+ "\n" \
+ "[db]\n" \
+ "postgres={\"db-id\":12173,\"db-last-system-id\":12168}\n"
+
+ #define TEST_MANIFEST_FILE \
+ "\n" \
+ "[target:file]\n" \
+ "pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"master\":true" \
+ ",\"size\":4,\"timestamp\":1565282114}\n"
+
+ #define TEST_MANIFEST_FILE_DEFAULT \
+ "\n" \
+ "[target:file:default]\n" \
+ "group=\"group1\"\n" \
+ "master=false\n" \
+ "mode=\"0600\"\n" \
+ "user=\"user1\"\n"
+
+ #define TEST_MANIFEST_LINK \
+ "\n" \
+ "[target:link]\n" \
+ "pg_data/pg_stat={\"destination\":\"../pg_stat\"}\n"
+
+ #define TEST_MANIFEST_LINK_DEFAULT \
+ "\n" \
+ "[target:link:default]\n" \
+ "group=\"group1\"\n" \
+ "user=false\n"
+
+ #define TEST_MANIFEST_PATH \
+ "\n" \
+ "[target:path]\n" \
+ "pg_data={\"user\":\"user1\"}\n" \
+
+ #define TEST_MANIFEST_PATH_DEFAULT \
+ "\n" \
+ "[target:path:default]\n" \
+ "group=false\n" \
+ "mode=\"0700\"\n" \
+ "user=\"user1\"\n"
+
+ // *****************************************************************************************************************************
+ if (testBegin("verifyManifestFile()"))
+ {
+ // Load Parameters
+ StringList *argList = strLstDup(argListBase);
+ harnessCfgLoad(cfgCmdVerify, argList);
+
+ const Buffer *contentLoad = harnessInfoChecksumZ
+ (
+ TEST_MANIFEST_HEADER
+ TEST_MANIFEST_DB_92
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ TEST_MANIFEST_FILE
+ TEST_MANIFEST_FILE_DEFAULT
+ TEST_MANIFEST_LINK
+ TEST_MANIFEST_LINK_DEFAULT
+ TEST_MANIFEST_PATH
+ TEST_MANIFEST_PATH_DEFAULT
+ );
+
+ Manifest *manifest = NULL;
+ String *backupLabel = strNew("20181119-152138F");
+ String *manifestFile = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabel));
+ String *manifestFileCopy = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFile));
+ unsigned int jobErrorTotal = 0;
+ VerifyBackupResult backupResult = {.backupLabel = strDup(backupLabel)};
+
+ InfoArchive *archiveInfo = NULL;
+ TEST_ASSIGN(archiveInfo, infoArchiveNewLoad(ioBufferReadNew(archiveInfoBase)), "archive.info");
+ InfoPg *infoPg = infoArchivePg(archiveInfo);
+
+ //--------------------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("manifest.copy exists, no manifest main, manifest db version not in history, not current db");
+
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write manifest db section mismatch");
+
+ backupResult.status = backupValid;
+ TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest");
+ TEST_RESULT_PTR(manifest, NULL, "manifest not set - pg version mismatch");
+ TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid");
+ harnessLogResult(
+ strZ(strNewFmt(
+ "P00 WARN: unable to open missing file '%s/%s/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT "' for read\n"
+ "P00 ERROR: [028]: '%s' may not be recoverable - PG data (id 1, version 9.2, system-id 6625592122879095702) is not "
+ "in the backup.info history, skipping",
+ testPath(), strZ(backupStanzaPath), strZ(backupLabel), strZ(backupLabel))));
+
+ //--------------------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("rerun test with db-system-id invalid and no main");
+
+ contentLoad = harnessInfoChecksumZ
+ (
+ TEST_MANIFEST_HEADER
+ "\n"
+ "[backup:db]\n"
+ "db-catalog-version=201409291\n"
+ "db-control-version=942\n"
+ "db-id=1\n"
+ "db-system-id=0\n"
+ "db-version=\"9.4\"\n"
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ TEST_MANIFEST_FILE
+ TEST_MANIFEST_FILE_DEFAULT
+ TEST_MANIFEST_LINK
+ TEST_MANIFEST_LINK_DEFAULT
+ TEST_MANIFEST_PATH
+ TEST_MANIFEST_PATH_DEFAULT
+ );
+
+ TEST_RESULT_VOID(storageRemoveP(storageTest, manifestFile), "remove main manifest");
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write manifest copy invalid system-id");
+
+ backupResult.status = backupValid;
+ TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest");
+ TEST_RESULT_PTR(manifest, NULL, "manifest not set - pg system-id mismatch");
+ TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid");
+ harnessLogResult(
+ strZ(strNewFmt(
+ "P00 WARN: unable to open missing file '%s/%s/%s/" BACKUP_MANIFEST_FILE "' for read\n"
+ "P00 WARN: %s/backup.manifest is missing or unusable, using copy\n"
+ "P00 ERROR: [028]: '%s' may not be recoverable - PG data (id 1, version 9.4, system-id 0) is not "
+ "in the backup.info history, skipping",
+ testPath(), strZ(backupStanzaPath), strZ(backupLabel), strZ(backupLabel), strZ(backupLabel))));
+
+ //--------------------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("rerun copy test with db-id invalid");
+
+ contentLoad = harnessInfoChecksumZ
+ (
+ TEST_MANIFEST_HEADER
+ "\n"
+ "[backup:db]\n"
+ "db-catalog-version=201409291\n"
+ "db-control-version=942\n"
+ "db-id=0\n"
+ "db-system-id=6625592122879095702\n"
+ "db-version=\"9.4\"\n"
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ TEST_MANIFEST_FILE
+ TEST_MANIFEST_FILE_DEFAULT
+ TEST_MANIFEST_LINK
+ TEST_MANIFEST_LINK_DEFAULT
+ TEST_MANIFEST_PATH
+ TEST_MANIFEST_PATH_DEFAULT
+ );
+
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write manifest copy invalid db-id");
+
+ backupResult.status = backupValid;
+ TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest");
+ TEST_RESULT_PTR(manifest, NULL, "manifest not set - pg db-id mismatch");
+ TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid");
+ harnessLogResult(
+ strZ(strNewFmt(
+ "P00 WARN: unable to open missing file '%s/%s/%s/" BACKUP_MANIFEST_FILE "' for read\n"
+ "P00 WARN: %s/backup.manifest is missing or unusable, using copy\n"
+ "P00 ERROR: [028]: '%s' may not be recoverable - PG data (id 0, version 9.4, system-id 6625592122879095702) is not "
+ "in the backup.info history, skipping",
+ testPath(), strZ(backupStanzaPath), strZ(backupLabel), strZ(backupLabel), strZ(backupLabel))));
+
+ //--------------------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("missing main manifest, errored copy");
+
+ backupResult.status = backupValid;
+ contentLoad = BUFSTRDEF(
+ "[backrest]\n"
+ "backrest-checksum=\"BOGUS\"\n"
+ "backrest-format=5\n"
+ "backrest-version=\"2.28\"\n");
+
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write invalid manifest copy");
+ TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, false, infoPg, &jobErrorTotal), "verify manifest");
+ TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid");
+ harnessLogResult(
+ strZ(strNewFmt(
+ "P00 WARN: unable to open missing file '%s/%s/%s/" BACKUP_MANIFEST_FILE "' for read\n"
+ "P00 WARN: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' "
+ STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, testPath(), strZ(backupStanzaPath), strZ(backupLabel),
+ strZ(backupLabel))));
+
+ //--------------------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("current backup true");
+
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write invalid manifest");
+
+ TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, true, infoPg, &jobErrorTotal), "verify manifest");
+ TEST_RESULT_PTR(manifest, NULL, "manifest not set");
+ TEST_RESULT_UINT(backupResult.status, backupInvalid, "manifest unusable - backup invalid");
+ harnessLogResult(
+ strZ(strNewFmt(
+ "P00 WARN: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' "
+ STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE "\n"
+ "P00 WARN: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS' "
+ STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(backupLabel), strZ(backupLabel))));
+
+ // Write a valid manifest with a manifest copy that is invalid
+ contentLoad = harnessInfoChecksumZ
+ (
+ TEST_MANIFEST_HEADER
+ TEST_MANIFEST_DB_94
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ TEST_MANIFEST_FILE
+ TEST_MANIFEST_FILE_DEFAULT
+ TEST_MANIFEST_LINK
+ TEST_MANIFEST_LINK_DEFAULT
+ TEST_MANIFEST_PATH
+ TEST_MANIFEST_PATH_DEFAULT
+ );
+
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write valid manifest");
+
+ backupResult.status = backupValid;
+ TEST_ASSIGN(manifest, verifyManifestFile(&backupResult, NULL, true, infoPg, &jobErrorTotal), "verify manifest");
+ TEST_RESULT_PTR_NE(manifest, NULL, "manifest set");
+ TEST_RESULT_UINT(backupResult.status, backupValid, "manifest usable");
+ harnessLogResult(strZ(strNewFmt("P00 WARN: backup '%s' manifest.copy does not match manifest", strZ(backupLabel))));
+ }
+
// *****************************************************************************************************************************
if (testBegin("verifyCreateArchiveIdRange()"))
{
@@ -378,15 +680,6 @@ testRun(void)
unsigned int errTotal = 0;
- TEST_RESULT_STR_Z(
- verifySetBackupCheckArchive(
- strLstNew(), backupInfo, strLstNew(), pgHistory, &errTotal), NULL, "no archives or backups");
- TEST_RESULT_UINT(errTotal, 0, "no error");
- TEST_RESULT_STR_Z(
- verifySetBackupCheckArchive(
- backupList, backupInfo, archiveIdList, pgHistory, &errTotal), NULL, "no current backup, no missing archive");
- TEST_RESULT_UINT(errTotal, 0, "no error");
-
// Add backup to end of list
strLstAddZ(backupList, "20181119-153000F");
strLstAddZ(archiveIdList, "12-3");
@@ -408,13 +701,17 @@ testRun(void)
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("verifyLogInvalidResult() - missing file");
- TEST_RESULT_UINT(verifyLogInvalidResult(verifyFileMissing, 0, strNew("missingfilename")), 0, "file missing message");
+ TEST_RESULT_UINT(
+ verifyLogInvalidResult(STORAGE_REPO_ARCHIVE_STR, verifyFileMissing, 0, strNew("missingfilename")),
+ 0, "file missing message");
harnessLogResult("P00 WARN: file missing 'missingfilename'");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("verifyRender() - missing file, empty invalidList");
List *archiveIdResultList = lstNewP(sizeof(VerifyArchiveResult), .comparator = archiveIdComparator);
+ List *backupResultList = lstNewP(sizeof(VerifyBackupResult), .comparator = lstComparatorStr);
+
VerifyArchiveResult archiveIdResult =
{
.archiveId = strNew("9.6-1"),
@@ -431,25 +728,40 @@ testRun(void)
lstAdd(archiveIdResult.walRangeList, &walRange);
lstAdd(archiveIdResultList, &archiveIdResult);
TEST_RESULT_STR_Z(
- verifyRender(archiveIdResultList),
+ verifyRender(archiveIdResultList, backupResultList),
"Results:\n"
" archiveId: 9.6-1, total WAL checked: 1, total valid WAL: 0\n"
- " missing: 0, checksum invalid: 0, size invalid: 0, other: 0", "no invalid file list");
+ " missing: 0, checksum invalid: 0, size invalid: 0, other: 0\n"
+ " backup: none found", "archive: no invalid file list");
VerifyInvalidFile invalidFile =
{
.fileName = strNew("file"),
.reason = verifyFileMissing,
};
-
lstAdd(walRange.invalidFileList, &invalidFile);
+
+ VerifyBackupResult backupResult =
+ {
+ .backupLabel = strNew("test-backup-label"),
+ .status = backupInvalid,
+ .totalFileVerify = 1,
+ .invalidFileList = lstNewP(sizeof(VerifyInvalidFile), .comparator = lstComparatorStr),
+ };
+ lstAdd(backupResult.invalidFileList, &invalidFile);
+ lstAdd(backupResultList, &backupResult);
+
TEST_RESULT_STR_Z(
- verifyRender(archiveIdResultList),
+ verifyRender(archiveIdResultList, backupResultList),
"Results:\n"
" archiveId: 9.6-1, total WAL checked: 1, total valid WAL: 0\n"
- " missing: 1, checksum invalid: 0, size invalid: 0, other: 0", "file missing");
+ " missing: 1, checksum invalid: 0, size invalid: 0, other: 0\n"
+ " backup: test-backup-label, status: invalid, total files checked: 1, total valid files: 0\n"
+ " missing: 1, checksum invalid: 0, size invalid: 0, other: 0", "archive file missing, backup file missing");
+
+ //--------------------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("verifyAddInvalidWalFile() - file missing (coverage test)");
- // Coverage test
TEST_RESULT_VOID(
verifyAddInvalidWalFile(archiveIdResult.walRangeList, verifyFileMissing, strNew("test"), strNew("3")), "coverage test");
}
@@ -461,20 +773,6 @@ testRun(void)
StringList *argList = strLstDup(argListBase);
harnessCfgLoad(cfgCmdVerify, argList);
- //--------------------------------------------------------------------------------------------------------------------------
- TEST_TITLE("neither backup nor archive info files exist");
-
- TEST_ERROR(cmdVerify(), RuntimeError, "2 fatal errors encountered, see log for details");
- harnessLogResult(
- strZ(strNewFmt(
- "P00 WARN: unable to open missing file '%s/%s/backup.info' for read\n"
- "P00 WARN: unable to open missing file '%s/%s/backup.info.copy' for read\n"
- "P00 ERROR: [029]: No usable backup.info file\n"
- "P00 WARN: unable to open missing file '%s/%s/archive.info' for read\n"
- "P00 WARN: unable to open missing file '%s/%s/archive.info.copy' for read\n"
- "P00 ERROR: [029]: No usable archive.info file", testPath(), strZ(backupStanzaPath), testPath(),
- strZ(backupStanzaPath), testPath(), strZ(archiveStanzaPath), testPath(), strZ(archiveStanzaPath))));
-
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("backup.info invalid checksum, neither backup copy nor archive infos exist");
@@ -575,34 +873,6 @@ testRun(void)
"P00 WARN: unable to open missing file '%s/%s/backup.info.copy' for read\n"
"P00 ERROR: [029]: No usable backup.info file", testPath(), strZ(backupStanzaPath), testPath(),
strZ(backupStanzaPath))));
-
- //--------------------------------------------------------------------------------------------------------------------------
- TEST_TITLE("backup.info.copy valid, archive.info and copy valid, present but empty backup");
-
- TEST_RESULT_VOID(
- storagePutP(storageNewWriteP(storageTest, backupInfoFileName), backupInfoMultiHistoryBase),
- "write valid backup.info");
- TEST_RESULT_VOID(
- storagePutP(storageNewWriteP(storageTest, backupInfoFileNameCopy), backupInfoMultiHistoryBase),
- "write valid backup.info.copy");
- TEST_RESULT_VOID(
- storagePathCreateP(storageTest, strNewFmt("%s/20200810-171426F", strZ(backupStanzaPath))),
- "create empty backup label path");
- TEST_RESULT_VOID(cmdVerify(), "no archives, empty backup label path");
- harnessLogResult("P00 WARN: no archives exist in the repo");
-
- //--------------------------------------------------------------------------------------------------------------------------
- TEST_TITLE("backup.info.copy valid, archive.info and copy valid, present but empty backup, empty archive");
-
- TEST_RESULT_VOID(
- storagePathCreateP(storageTest, strNewFmt("%s/9.4-1", strZ(archiveStanzaPath))),
- "create empty path for archiveId");
-
- TEST_RESULT_VOID(cmdVerify(), "no jobs - empty archive id and backup label paths");
- harnessLogResult(
- "P00 WARN: archive path '9.4-1' is empty\n"
- "P00 INFO: Results:\n"
- " archiveId: 9.4-1, total WAL checked: 0, total valid WAL: 0");
}
// *****************************************************************************************************************************
@@ -619,19 +889,12 @@ testRun(void)
TEST_RESULT_VOID(storagePutP(storageNewWriteP(storageRepoWrite(), filePathName), BUFSTRDEF("")), "put zero-sized file");
TEST_RESULT_UINT(verifyFile(filePathName, STRDEF(HASH_TYPE_SHA1_ZERO), 0, NULL), verifyOk, "file ok");
- const char *fileContents = "acefile";
- uint64_t fileSize = 7;
- const String *checksum = STRDEF("d1cd8a7d11daa26814b93eb604e1d49ab4b43770");
-
TEST_RESULT_VOID(storagePutP(storageNewWriteP(storageRepoWrite(), filePathName), BUFSTRZ(fileContents)), "put file");
- TEST_RESULT_UINT(verifyFile(filePathName, checksum, fileSize, NULL), verifyOk, "file size ok");
- TEST_RESULT_UINT(verifyFile(filePathName, checksum, 0, NULL), verifySizeInvalid, "file size invalid");
- TEST_RESULT_UINT(
- verifyFile(filePathName, strNew("badchecksum"), fileSize, NULL), verifyChecksumMismatch, "file checksum mismatch");
+ TEST_RESULT_UINT(verifyFile(filePathName, fileChecksum, 0, NULL), verifySizeInvalid, "file size invalid");
TEST_RESULT_UINT(
verifyFile(
- strNewFmt(STORAGE_REPO_ARCHIVE "/missingFile"), checksum, 0, NULL), verifyFileMissing, "file missing");
+ strNewFmt(STORAGE_REPO_ARCHIVE "/missingFile"), fileChecksum, 0, NULL), verifyFileMissing, "file missing");
// Create a compressed encrypted repo file
filePathName = strNew(STORAGE_REPO_BACKUP "/testfile.gz");
@@ -642,7 +905,7 @@ testRun(void)
TEST_RESULT_VOID(storagePutP(write, BUFSTRZ(fileContents)), "write encrypted, compressed file");
TEST_RESULT_UINT(
- verifyFile(filePathName, checksum, fileSize, strNew("pass")), verifyOk, "file encrypted compressed ok");
+ verifyFile(filePathName, fileChecksum, fileSize, strNew("pass")), verifyOk, "file encrypted compressed ok");
TEST_RESULT_UINT(
verifyFile(
filePathName, strNew("badchecksum"), fileSize, strNew("pass")), verifyChecksumMismatch,
@@ -660,7 +923,7 @@ testRun(void)
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(filePathName));
- varLstAdd(paramList, varNewStr(checksum));
+ varLstAdd(paramList, varNewStr(fileChecksum));
varLstAdd(paramList, varNewUInt64(fileSize));
varLstAdd(paramList, varNewStrZ("pass"));
@@ -672,7 +935,7 @@ testRun(void)
}
// *****************************************************************************************************************************
- if (testBegin("cmdVerify(), verifyProcess()"))
+ if (testBegin("cmdVerify(), verifyProcess() - errors"))
{
//--------------------------------------------------------------------------------------------------------------------------
StringList *argList = strLstDup(argListBase);
@@ -697,18 +960,14 @@ testRun(void)
storagePutP(storageNewWriteP(storageTest, backupInfoFileName),
harnessInfoChecksumZ(
"[db]\n"
- "db-catalog-version=201707211\n"
- "db-control-version=1100\n"
- "db-id=2\n"
- "db-system-id=6626363367545678089\n"
- "db-version=\"11\"\n"
+ TEST_BACKUP_DB2_11
"\n"
"[db:history]\n"
- "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702,"
- "\"db-version\":\"9.4\"}\n"
- "2={\"db-catalog-version\":201707211,\"db-control-version\":1100,\"db-system-id\":6626363367545678089,"
- "\"db-version\":\"11\"}")),
- "put backup.info files");
+ TEST_BACKUP_DB1_HISTORY
+ "\n"
+ TEST_BACKUP_DB2_HISTORY
+ )),
+ "put backup.info files - no current backups");
storageCopy(storageNewReadP(storageTest, backupInfoFileName), storageNewWriteP(storageTest, backupInfoFileNameCopy));
//--------------------------------------------------------------------------------------------------------------------------
@@ -743,13 +1002,14 @@ testRun(void)
TEST_ERROR(cmdVerify(), RuntimeError, "1 fatal errors encountered, see log for details");
harnessLogResult(
- strZ(strNewFmt(
+ strZ(strNew(
"P00 WARN: no backups exist in the repo\n"
"P00 ERROR: [028]: duplicate WAL '000000020000000700000FFE' for '11-2' exists, skipping\n"
"P00 WARN: path '11-2/0000000200000007' does not contain any valid WAL to be processed\n"
"P00 INFO: Results:\n"
" archiveId: 11-2, total WAL checked: 2, total valid WAL: 0\n"
- " missing: 0, checksum invalid: 0, size invalid: 0, other: 0")));
+ " missing: 0, checksum invalid: 0, size invalid: 0, other: 0\n"
+ " backup: none found")));
harnessLogLevelReset();
@@ -797,14 +1057,16 @@ testRun(void)
unsigned int errorTotal = 0;
TEST_RESULT_STR_Z(
verifyProcess(&errorTotal),
- "Results:\n"
- " archiveId: 9.4-1, total WAL checked: 0, total valid WAL: 0\n"
- " archiveId: 11-2, total WAL checked: 4, total valid WAL: 2\n"
- " missing: 0, checksum invalid: 1, size invalid: 1, other: 0",
+ strZ(strNew(
+ "Results:\n"
+ " archiveId: 9.4-1, total WAL checked: 0, total valid WAL: 0\n"
+ " archiveId: 11-2, total WAL checked: 4, total valid WAL: 2\n"
+ " missing: 0, checksum invalid: 1, size invalid: 1, other: 0\n"
+ " backup: none found")),
"verifyProcess() results");
TEST_RESULT_UINT(errorTotal, 2, "errors");
harnessLogResult(
- strZ(strNewFmt(
+ strZ(strNew(
"P00 WARN: no backups exist in the repo\n"
"P00 WARN: archive path '9.4-1' is empty\n"
"P00 WARN: path '11-2/0000000100000000' does not contain any valid WAL to be processed\n"
@@ -848,14 +1110,14 @@ testRun(void)
TEST_ERROR(cmdVerify(), RuntimeError, "2 fatal errors encountered, see log for details");
harnessLogResult(
- strZ(strNewFmt(
+ strZ(strNew(
"P01 ERROR: [028]: invalid checksum "
"'11-2/0000000200000007/000000020000000700000FFD-a6e1a64f0813352bc2e97f116a1800377e17d2e4.gz'\n"
"P01 ERROR: [028]: invalid size "
"'11-2/0000000200000007/000000020000000700000FFF-ee161f898c9012dd0c28b3fd1e7140b9cf411306'")));
//--------------------------------------------------------------------------------------------------------------------------
- TEST_TITLE("valid info files, unreadable WAL file");
+ TEST_TITLE("valid info files - various archive/backup errors");
// Load Parameters - single non-default repo
argList = strLstNew();
@@ -874,41 +1136,440 @@ testRun(void)
walBuffer),
"write WAL - file not readable");
+ String *backupLabelPriorNoManifest = strNew("20181119-152800F");
+ TEST_RESULT_VOID(
+ storagePathCreateP(storageTest, strNewFmt("%s/%s", strZ(backupStanzaPath), strZ(backupLabelPriorNoManifest))),
+ "prior backup path missing manifests");
+
+ String *backupLabelManifestNoTargetFile = strNew("20181119-152810F");
+ const Buffer *contentLoad = harnessInfoChecksumZ
+ (
+ TEST_MANIFEST_HEADER
+ TEST_MANIFEST_DB_94
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ );
+
+ String *manifestFileNoTarget = strNewFmt(
+ "%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabelManifestNoTargetFile));
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileNoTarget), contentLoad), "write manifest without target files");
+
+ // Create full backup with files
+ String *backupLabel = strNew("20181119-152900F");
+
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, strNewFmt("%s/%s/pg_data/PG_VERSION", strZ(backupStanzaPath),
+ strZ(backupLabel))), BUFSTRDEF("BOGUS")), "put checksum-error backup file");
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, strNewFmt("%s/%s/pg_data/testzero", strZ(backupStanzaPath),
+ strZ(backupLabel))), BUFSTRDEF("")), "put zero-size backup file");
+
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, strNewFmt("%s/%s/pg_data/testvalid", strZ(backupStanzaPath),
+ strZ(backupLabel))), BUFSTRZ(fileContents)), "put valid file");
+
+ contentLoad = harnessInfoChecksumZ
+ (
+ strZ(strNewFmt(
+ "[backup]\n"
+ "backup-label=\"%s\"\n"
+ "backup-timestamp-copy-start=0\n"
+ "backup-timestamp-start=0\n"
+ "backup-timestamp-stop=0\n"
+ "backup-type=\"full\"\n"
+ "\n"
+ "[backup:db]\n"
+ TEST_BACKUP_DB2_11
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ TEST_MANIFEST_FILE
+ "pg_data/testvalid={\"checksum\":\"%s\",\"master\":true,\"size\":7,\"timestamp\":1565282114}\n"
+ "pg_data/testzero={\"repo-size\":20,\"size\":0,\"timestamp\":1601405663}\n"
+ TEST_MANIFEST_FILE_DEFAULT
+ TEST_MANIFEST_LINK
+ TEST_MANIFEST_LINK_DEFAULT
+ TEST_MANIFEST_PATH
+ TEST_MANIFEST_PATH_DEFAULT,
+ strZ(backupLabel), strZ(fileChecksum)))
+ );
+
+ // Write manifests for full backup
+ String *manifestFile = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabel));
+ String *manifestFileCopy = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFile));
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write valid manifest");
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad),
+ "write valid manifest copy");
+
+ // Create a manifest for the dependent that has references
+ String *backupLabelDependent = strNew("20181119-152900F_20181119-152909D");
+
+ // Create an unprocessed backup label with a file that will be referenced in this manifest
+ String *unprocessedBackup = strNew("UNPROCESSEDBACKUP");
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, strNewFmt("%s/%s/pg_data/testother", strZ(backupStanzaPath),
+ strZ(unprocessedBackup)), .modeFile = 0200), BUFSTRZ(fileContents)), "put unreadable file to unprocessed backup");
+
+ contentLoad = harnessInfoChecksumZ
+ (
+ strZ(strNewFmt(
+ "[backup]\n"
+ "backup-label=\"%s\"\n"
+ "backup-timestamp-copy-start=0\n"
+ "backup-timestamp-start=0\n"
+ "backup-timestamp-stop=0\n"
+ "backup-type=\"diff\"\n"
+ "\n"
+ "[backup:db]\n"
+ TEST_BACKUP_DB2_11
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ "\n"
+ "[target:file]\n"
+ "pg_data/PG_VERSION="
+ "{\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"master\":true,\"reference\":\"%s\",\"size\":4,"
+ "\"timestamp\":1565282114}\n"
+ "pg_data/testfile={\"checksum\":\"%s\",\"master\":true,\"reference\":\"%s\",\"size\":7,\"timestamp\":1565282114}\n"
+ "pg_data/testfile2={\"checksum\":\"%s\",\"master\":true,\"size\":7,\"timestamp\":1565282114}\n"
+ "pg_data/testmissing="
+ "{\"checksum\":\"123473f470864e067ee3a22e64b47b0a1c356abc\",\"size\":7,\"timestamp\":1565282114}\n"
+ "pg_data/testother={\"checksum\":\"%s\",\"master\":true,\"reference\":\"%s\",\"size\":7,\"timestamp\":1565282114}\n"
+ TEST_MANIFEST_FILE_DEFAULT
+ TEST_MANIFEST_LINK
+ TEST_MANIFEST_LINK_DEFAULT
+ TEST_MANIFEST_PATH
+ TEST_MANIFEST_PATH_DEFAULT,
+ strZ(backupLabelDependent), strZ(backupLabel), strZ(fileChecksum), strZ(backupLabel), strZ(fileChecksum),
+ strZ(fileChecksum), strZ(unprocessedBackup)))
+ );
+
+ // Write manifests for dependent backup
+ manifestFile = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabelDependent));
+ manifestFileCopy = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFile));
+
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write manifest to dependent");
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write manifest copy to dependent");
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, strNewFmt("%s/%s/pg_data/testfile2", strZ(backupStanzaPath),
+ strZ(backupLabelDependent))), BUFSTRZ(fileContents)), "put valid file to dependent");
+
+ // Create in-progress backup
+ TEST_RESULT_VOID(
+ storagePathCreateP(storageTest, strNewFmt("%s/%s", strZ(backupStanzaPath), "20181119-153000F")),
+ "create empty backup path for newest backup so in-progress");
+
// Set log level to capture ranges
harnessLogLevelSet(logLevelDetail);
- TEST_ERROR(cmdVerify(), RuntimeError, "3 fatal errors encountered, see log for details");
+ TEST_ERROR(cmdVerify(), RuntimeError, "7 fatal errors encountered, see log for details");
harnessLogResult(
strZ(strNewFmt(
- "P00 WARN: no backups exist in the repo\n"
"P00 WARN: archive path '9.4-1' is empty\n"
"P00 WARN: path '11-2/0000000100000000' does not contain any valid WAL to be processed\n"
"P01 ERROR: [028]: invalid checksum "
"'11-2/0000000200000007/000000020000000700000FFD-a6e1a64f0813352bc2e97f116a1800377e17d2e4.gz'\n"
"P01 ERROR: [028]: invalid size "
"'11-2/0000000200000007/000000020000000700000FFF-ee161f898c9012dd0c28b3fd1e7140b9cf411306'\n"
- "P01 ERROR: [039]: invalid verify "
+ "P01 ERROR: [039]: invalid result "
"11-2/0000000200000008/000000020000000800000003-656817043007aa2100c44c712bcb456db705dab9: [41] raised from "
"local-1 protocol: unable to open file "
"'%s/%s/11-2/0000000200000008/000000020000000800000003-656817043007aa2100c44c712bcb456db705dab9' for read: "
"[13] Permission denied\n"
+ "P00 WARN: unable to open missing file '%s/%s/20181119-152800F/backup.manifest' for read\n"
+ "P00 WARN: unable to open missing file '%s/%s/20181119-152800F/backup.manifest.copy' for read\n"
+ "P00 WARN: manifest missing for '20181119-152800F' - backup may have expired\n"
+ "P00 WARN: unable to open missing file '%s/%s/20181119-152810F/backup.manifest.copy' for read\n"
+ "P00 ERROR: [028]: backup '20181119-152810F' manifest does not contain any target files to verify\n"
+ "P01 ERROR: [028]: invalid checksum '20181119-152900F/pg_data/PG_VERSION'\n"
+ "P01 ERROR: [028]: file missing '20181119-152900F_20181119-152909D/pg_data/testmissing'\n"
+ "P00 WARN: unable to open missing file '%s/%s/20181119-153000F/backup.manifest' for read\n"
+ "P00 INFO: backup '20181119-153000F' appears to be in progress, skipping\n"
+ "P01 ERROR: [039]: invalid result UNPROCESSEDBACKUP/pg_data/testother: [41] raised from local-1 protocol: unable "
+ "to open file '%s/%s/UNPROCESSEDBACKUP/pg_data/testother' for read: [13] Permission denied\n"
"P00 DETAIL: archiveId: 11-2, wal start: 000000020000000700000FFD, wal stop: 000000020000000800000000\n"
"P00 DETAIL: archiveId: 11-2, wal start: 000000020000000800000002, wal stop: 000000020000000800000003\n"
"P00 DETAIL: archiveId: 11-2, wal start: 000000030000000000000000, wal stop: 000000030000000000000001\n"
"P00 INFO: Results:\n"
" archiveId: 9.4-1, total WAL checked: 0, total valid WAL: 0\n"
" archiveId: 11-2, total WAL checked: 8, total valid WAL: 5\n"
- " missing: 0, checksum invalid: 1, size invalid: 1, other: 1",
- testPath(), strZ(archiveStanzaPath))));
+ " missing: 0, checksum invalid: 1, size invalid: 1, other: 1\n"
+ " backup: 20181119-152800F, status: manifest missing, total files checked: 0, total valid files: 0\n"
+ " backup: 20181119-152810F, status: invalid, total files checked: 0, total valid files: 0\n"
+ " backup: 20181119-152900F, status: invalid, total files checked: 3, total valid files: 2\n"
+ " missing: 0, checksum invalid: 1, size invalid: 0, other: 0\n"
+ " backup: 20181119-152900F_20181119-152909D, status: invalid, total files checked: 5, "
+ "total valid files: 2\n"
+ " missing: 1, checksum invalid: 1, size invalid: 0, other: 1\n"
+ " backup: 20181119-153000F, status: in-progress, total files checked: 0, total valid files: 0",
+ testPath(), strZ(archiveStanzaPath), testPath(), strZ(backupStanzaPath), testPath(), strZ(backupStanzaPath),
+ testPath(), strZ(backupStanzaPath), testPath(), strZ(backupStanzaPath), testPath(), strZ(backupStanzaPath))));
harnessLogLevelReset();
+ }
+
+ // *****************************************************************************************************************************
+ if (testBegin("cmdVerify()"))
+ {
+ // Load Parameters
+ StringList *argList = strLstDup(argListBase);
+ harnessCfgLoad(cfgCmdVerify, argList);
+
+ // Backup labels
+ String *backupLabelFull = strNew("20181119-152900F");
+ String *backupLabelDiff = strNew("20181119-152900F_20181119-152909D");
+ String *backupLabelFullDb2 = strNew("20201119-163000F");
+
+ #define TEST_BACKUP_DB1_CURRENT_FULL3_DIFF1 \
+ "20181119-152900F_20181119-152909D={" \
+ "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," \
+ "\"backup-archive-start\":\"000000010000000000000006\",\"backup-archive-stop\":\"000000010000000000000007\"," \
+ "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \
+ "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \
+ "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," \
+ "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \
+ "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
+
+ #define TEST_BACKUP_DB2_CURRENT_FULL1 \
+ "20201119-163000F={" \
+ "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," \
+ "\"backup-archive-start\":\"000000020000000000000001\",\"backup-archive-stop\":\"000000020000000000000001\"," \
+ "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," \
+ "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," \
+ "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," \
+ "\"db-id\":2,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," \
+ "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
+
+ //--------------------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("prior backup verification incomplete - referenced file checked");
TEST_RESULT_VOID(
- storageRemoveP(
- storageTest,
- strNewFmt("%s/11-2/0000000200000008/000000020000000800000003-656817043007aa2100c44c712bcb456db705dab9",
- strZ(archiveStanzaPath))),
- "remove unreadable WAL");
+ storagePutP(storageNewWriteP(storageTest, archiveInfoFileName), archiveInfoMultiHistoryBase),
+ "write archive.info");
+ storageCopy(storageNewReadP(storageTest, archiveInfoFileName), storageNewWriteP(storageTest, archiveInfoFileNameCopy));
+
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, backupInfoFileName),
+ harnessInfoChecksumZ(
+ "[backup:current]\n"
+ TEST_BACKUP_DB1_CURRENT_FULL3
+ TEST_BACKUP_DB1_CURRENT_FULL3_DIFF1
+ TEST_BACKUP_DB2_CURRENT_FULL1
+ "\n"
+ "[db]\n"
+ TEST_BACKUP_DB2_11
+ "\n"
+ "[db:history]\n"
+ TEST_BACKUP_DB1_HISTORY
+ "\n"
+ TEST_BACKUP_DB2_HISTORY
+ )),
+ "write backup.info");
+ storageCopy(storageNewReadP(storageTest, backupInfoFileName), storageNewWriteP(storageTest, backupInfoFileNameCopy));
+
+ // Create valid full backup and valid diff backup for DB1
+ const Buffer *contentLoad = harnessInfoChecksumZ
+ (
+ TEST_MANIFEST_HEADER
+ TEST_MANIFEST_DB_94
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ TEST_MANIFEST_FILE
+ TEST_MANIFEST_FILE_DEFAULT
+ TEST_MANIFEST_LINK
+ TEST_MANIFEST_LINK_DEFAULT
+ TEST_MANIFEST_PATH
+ TEST_MANIFEST_PATH_DEFAULT
+ );
+
+ // Write manifests for full backup
+ String *manifestFile = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabelFull));
+ String *manifestFileCopy = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFile));
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write valid manifest - full");
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write valid manifest copy - full");
+
+ contentLoad = harnessInfoChecksumZ
+ (
+ strZ(strNewFmt(
+ TEST_MANIFEST_HEADER
+ TEST_MANIFEST_DB_94
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ "\n"
+ "[target:file]\n"
+ "pg_data/PG_VERSION="
+ "{\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"master\":true,\"reference\":\"%s\",\"size\":4,"
+ "\"timestamp\":1565282114}\n"
+ TEST_MANIFEST_FILE_DEFAULT
+ TEST_MANIFEST_LINK
+ TEST_MANIFEST_LINK_DEFAULT
+ TEST_MANIFEST_PATH
+ TEST_MANIFEST_PATH_DEFAULT,
+ strZ(backupLabelFull)))
+ );
+
+ // Write manifests for diff backup
+ String *manifestFileDiff = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabelDiff));
+ String *manifestFileCopyDiff = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFileDiff));
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileDiff), contentLoad), "write valid manifest - diff");
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileCopyDiff), contentLoad), "write valid manifest copy - diff");
+
+ // Put the file referenced by both backups into the full backup
+ String *filePathName = strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION", strZ(backupLabelFull));
+ TEST_RESULT_VOID(storagePutP(storageNewWriteP(storageRepoWrite(), filePathName), BUFSTRZ(fileContents)), "put file");
+
+ TEST_ERROR(cmdVerify(), RuntimeError, "2 fatal errors encountered, see log for details");
+
+ // The error for the referenced file is logged twice because it is checked again by the second backup since the first backup
+ // verification had not yet completed before the second backup verification began
+ harnessLogResult(
+ strZ(strNewFmt(
+ "P00 WARN: no archives exist in the repo\n"
+ "P01 ERROR: [028]: invalid checksum '%s/pg_data/PG_VERSION'\n"
+ "P01 ERROR: [028]: invalid checksum '%s/pg_data/PG_VERSION'\n"
+ "P00 INFO: Results:\n"
+ " archiveId: none found\n"
+ " backup: %s, status: invalid, total files checked: 1, total valid files: 0\n"
+ " missing: 0, checksum invalid: 1, size invalid: 0, other: 0\n"
+ " backup: %s, status: invalid, total files checked: 1, total valid files: 0\n"
+ " missing: 0, checksum invalid: 1, size invalid: 0, other: 0",
+ strZ(backupLabelFull), strZ(backupLabelFull), strZ(backupLabelFull), strZ(backupLabelDiff))));
+
+ //--------------------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("valid backup, prior backup verification complete - referenced file not checked");
+
+ // Set process max to 1 and add more files to check so first backup completes before second is checked
+ strLstAddZ(argList, "--process-max=1");
+ harnessCfgLoad(cfgCmdVerify, argList);
+
+ contentLoad = harnessInfoChecksumZ
+ (
+ strZ(strNewFmt(
+ TEST_MANIFEST_HEADER
+ TEST_MANIFEST_DB_94
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ TEST_MANIFEST_FILE
+ "pg_data/base/1/555_init="
+ "{\"checksum\":\"%s\",\"master\":false,\"size\":1,\"timestamp\":1565282114}\n"
+ "pg_data/base/1/555_init.1={\"master\":false,\"size\":0,\"timestamp\":1565282114}\n"
+ TEST_MANIFEST_FILE_DEFAULT
+ TEST_MANIFEST_LINK
+ TEST_MANIFEST_LINK_DEFAULT
+ TEST_MANIFEST_PATH
+ TEST_MANIFEST_PATH_DEFAULT,
+ strZ(fileChecksum)))
+ );
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write valid manifest - full");
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write valid manifest copy - full");
+ filePathName = strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/base/1/555_init", strZ(backupLabelFull));
+ TEST_RESULT_VOID(storagePutP(storageNewWriteP(storageRepoWrite(), filePathName), BUFSTRZ(fileContents)),
+ "put file - invalid size");
+
+ contentLoad = harnessInfoChecksumZ
+ (
+ strZ(strNewFmt(
+ TEST_MANIFEST_HEADER
+ TEST_MANIFEST_DB_94
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ "\n"
+ "[target:file]\n"
+ "pg_data/PG_VERSION="
+ "{\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"master\":true,\"reference\":\"%s\",\"size\":4,"
+ "\"timestamp\":1565282114}\n"
+ TEST_MANIFEST_FILE_DEFAULT
+ TEST_MANIFEST_LINK
+ TEST_MANIFEST_LINK_DEFAULT
+ TEST_MANIFEST_PATH
+ TEST_MANIFEST_PATH_DEFAULT,
+ strZ(backupLabelFull)))
+ );
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileDiff), contentLoad), "write valid manifest - diff");
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileCopyDiff), contentLoad), "write valid manifest copy - diff");
+
+ // Create valid full backup and valid diff backup
+ contentLoad = harnessInfoChecksumZ
+ (
+ strZ(strNewFmt(
+ TEST_MANIFEST_HEADER
+ "\n"
+ "[backup:db]\n"
+ TEST_BACKUP_DB2_11
+ TEST_MANIFEST_OPTION_ALL
+ TEST_MANIFEST_TARGET
+ TEST_MANIFEST_DB
+ "\n"
+ "[target:file]\n"
+ "pg_data/validfile={\"checksum\":\"%s\",\"master\":true,\"size\":%u,\"timestamp\":1565282114}\n"
+ TEST_MANIFEST_FILE_DEFAULT
+ TEST_MANIFEST_LINK
+ TEST_MANIFEST_LINK_DEFAULT
+ TEST_MANIFEST_PATH
+ TEST_MANIFEST_PATH_DEFAULT,
+ strZ(fileChecksum), (unsigned int)fileSize))
+ );
+
+ manifestFile = strNewFmt("%s/%s/" BACKUP_MANIFEST_FILE, strZ(backupStanzaPath), strZ(backupLabelFullDb2));
+ manifestFileCopy = strNewFmt("%s" INFO_COPY_EXT, strZ(manifestFile));
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFile), contentLoad), "write valid manifest - full");
+ TEST_RESULT_VOID(
+ storagePutP(storageNewWriteP(storageTest, manifestFileCopy), contentLoad), "write valid manifest copy - full");
+ filePathName = strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/validfile", strZ(backupLabelFullDb2));
+ TEST_RESULT_VOID(storagePutP(storageNewWriteP(storageRepoWrite(), filePathName), BUFSTRZ(fileContents)), "put valid file");
+
+ // Create WAL file with just header info and small WAL size
+ Buffer *walBuffer = bufNew((size_t)(1024 * 1024));
+ bufUsedSet(walBuffer, bufSize(walBuffer));
+ memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
+ pgWalTestToBuffer(
+ (PgWal){.version = PG_VERSION_11, .systemId = 6626363367545678089, .size = 1024 * 1024}, walBuffer);
+ const char *walBufferSha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer)));
+ TEST_RESULT_VOID(
+ storagePutP(
+ storageNewWriteP(
+ storageTest,
+ strNewFmt("%s/11-2/0000000200000000/000000020000000000000001-%s", strZ(archiveStanzaPath), walBufferSha1)),
+ walBuffer),
+ "write valid WAL");
+
+ TEST_ERROR(cmdVerify(), RuntimeError, "3 fatal errors encountered, see log for details");
+
+ harnessLogResult(
+ strZ(strNewFmt(
+ "P01 ERROR: [028]: invalid checksum '%s/pg_data/PG_VERSION'\n"
+ "P01 ERROR: [028]: invalid size '%s/pg_data/base/1/555_init'\n"
+ "P01 ERROR: [028]: file missing '%s/pg_data/base/1/555_init.1'\n"
+ "P00 INFO: Results:\n"
+ " archiveId: 11-2, total WAL checked: 1, total valid WAL: 1\n"
+ " missing: 0, checksum invalid: 0, size invalid: 0, other: 0\n"
+ " backup: %s, status: invalid, total files checked: 3, total valid files: 0\n"
+ " missing: 1, checksum invalid: 1, size invalid: 1, other: 0\n"
+ " backup: %s, status: invalid, total files checked: 1, total valid files: 0\n"
+ " missing: 0, checksum invalid: 1, size invalid: 0, other: 0\n"
+ " backup: %s, status: valid, total files checked: 1, total valid files: 1\n"
+ " missing: 0, checksum invalid: 0, size invalid: 0, other: 0",
+ strZ(backupLabelFull), strZ(backupLabelFull), strZ(backupLabelFull), strZ(backupLabelFull),
+ strZ(backupLabelDiff), strZ(backupLabelFullDb2))));
}
FUNCTION_HARNESS_RESULT_VOID();