diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 3329fada8..cee6838f3 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -50,6 +50,7 @@ + diff --git a/src/Makefile.in b/src/Makefile.in index fa40ab72c..2af482b97 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -82,8 +82,8 @@ SRCS = \ command/repo/ls.c \ command/repo/put.c \ command/repo/rm.c \ + command/restore/blockChecksum.c \ command/restore/blockDelta.c \ - command/restore/blockHash.c \ command/restore/file.c \ command/restore/protocol.c \ command/restore/restore.c \ diff --git a/src/command/restore/blockHash.c b/src/command/restore/blockChecksum.c similarity index 66% rename from src/command/restore/blockHash.c rename to src/command/restore/blockChecksum.c index e790bb9bb..cd89d643b 100644 --- a/src/command/restore/blockHash.c +++ b/src/command/restore/blockChecksum.c @@ -1,9 +1,9 @@ /*********************************************************************************************************************************** -Restore Delta Map +Block Hash List ***********************************************************************************************************************************/ #include "build.auto.h" -#include "command/restore/blockHash.h" +#include "command/restore/blockChecksum.h" #include "common/crypto/common.h" #include "common/crypto/xxhash.h" #include "common/debug.h" @@ -13,35 +13,35 @@ Restore Delta Map /*********************************************************************************************************************************** Object type ***********************************************************************************************************************************/ -typedef struct BlockHash +typedef struct BlockChecksum { MemContext *memContext; // Mem context of filter size_t blockSize; // Block size for checksums size_t checksumSize; // Checksum size size_t blockCurrent; // Size of current block - IoFilter *hash; // Hash of current block - List *list; // List of hashes -} BlockHash; + IoFilter *checksum; // Checksum of current block + List *list; // List of checksums +} BlockChecksum; /*********************************************************************************************************************************** Macros for function logging ***********************************************************************************************************************************/ -#define FUNCTION_LOG_BLOCK_HASH_TYPE \ - BlockHash * -#define FUNCTION_LOG_BLOCK_HASH_FORMAT(value, buffer, bufferSize) \ - objNameToLog(value, "BlockHash", buffer, bufferSize) +#define FUNCTION_LOG_BLOCK_CHECKSUM_TYPE \ + BlockChecksum * +#define FUNCTION_LOG_BLOCK_CHECKSUM_FORMAT(value, buffer, bufferSize) \ + objNameToLog(value, "BlockChecksum", buffer, bufferSize) /*********************************************************************************************************************************** -Generate block hash list +Generate block checksum list ***********************************************************************************************************************************/ static void -blockHashProcess(THIS_VOID, const Buffer *const input) +blockChecksumProcess(THIS_VOID, const Buffer *const input) { - THIS(BlockHash); + THIS(BlockChecksum); FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(BLOCK_HASH, this); + FUNCTION_LOG_PARAM(BLOCK_CHECKSUM, this); FUNCTION_LOG_PARAM(BUFFER, input); FUNCTION_LOG_END(); @@ -53,37 +53,37 @@ blockHashProcess(THIS_VOID, const Buffer *const input) // Loop until input is consumed while (inputOffset != bufUsed(input)) { - // Create hash object if needed - if (this->hash == NULL) + // Create checksum object if needed + if (this->checksum == NULL) { MEM_CONTEXT_BEGIN(this->memContext) { - this->hash = xxHashNew(this->checksumSize); + this->checksum = xxHashNew(this->checksumSize); this->blockCurrent = 0; } MEM_CONTEXT_END(); } - // Calculate how much data to hash and perform hash + // Calculate how much data to checksum and perform checksum const size_t blockRemains = this->blockSize - this->blockCurrent; const size_t inputRemains = bufUsed(input) - inputOffset; - const size_t blockHash = blockRemains < inputRemains ? blockRemains : inputRemains; + const size_t blockChecksumSize = blockRemains < inputRemains ? blockRemains : inputRemains; - ioFilterProcessIn(this->hash, BUF(bufPtrConst(input) + inputOffset, blockHash)); + ioFilterProcessIn(this->checksum, BUF(bufPtrConst(input) + inputOffset, blockChecksumSize)); - // Update amount of data hashed - inputOffset += blockHash; - this->blockCurrent += blockHash; + // Update amount of data checksummed + inputOffset += blockChecksumSize; + this->blockCurrent += blockChecksumSize; - // If the block size has been reached then output the hash + // If the block size has been reached then output the checksum if (this->blockCurrent == this->blockSize) { MEM_CONTEXT_TEMP_BEGIN() { - lstAdd(this->list, bufPtrConst(pckReadBinP(pckReadNew(ioFilterResult(this->hash))))); + lstAdd(this->list, bufPtrConst(pckReadBinP(pckReadNew(ioFilterResult(this->checksum))))); - ioFilterFree(this->hash); - this->hash = NULL; + ioFilterFree(this->checksum); + this->checksum = NULL; } MEM_CONTEXT_TEMP_END(); } @@ -93,15 +93,15 @@ blockHashProcess(THIS_VOID, const Buffer *const input) } /*********************************************************************************************************************************** -Get a binary representation of the hash list +Get a binary representation of the checksum list ***********************************************************************************************************************************/ static Pack * -blockHashResult(THIS_VOID) +blockChecksumResult(THIS_VOID) { - THIS(BlockHash); + THIS(BlockChecksum); FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(BLOCK_HASH, this); + FUNCTION_LOG_PARAM(BLOCK_CHECKSUM, this); FUNCTION_LOG_END(); ASSERT(this != NULL); @@ -112,9 +112,9 @@ blockHashResult(THIS_VOID) { PackWrite *const packWrite = pckWriteNewP(); - // If there is a remainder in the hash - if (this->hash) - lstAdd(this->list, bufPtrConst(pckReadBinP(pckReadNew(ioFilterResult(this->hash))))); + // If there is a remainder in the checksum + if (this->checksum) + lstAdd(this->list, bufPtrConst(pckReadBinP(pckReadNew(ioFilterResult(this->checksum))))); pckWriteBinP(packWrite, BUF(lstGet(this->list, 0), lstSize(this->list) * this->checksumSize)); pckWriteEndP(packWrite); @@ -128,7 +128,7 @@ blockHashResult(THIS_VOID) /**********************************************************************************************************************************/ FN_EXTERN IoFilter * -blockHashNew(const size_t blockSize, const size_t checksumSize) +blockChecksumNew(const size_t blockSize, const size_t checksumSize) { FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_PARAM(SIZE, blockSize); @@ -140,11 +140,11 @@ blockHashNew(const size_t blockSize, const size_t checksumSize) // Allocate memory to hold process state IoFilter *this = NULL; - OBJ_NEW_BEGIN(BlockHash, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1) + OBJ_NEW_BEGIN(BlockChecksum, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1) { - BlockHash *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), IoFilter::BlockHash); + BlockChecksum *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), IoFilter::BlockChecksum); - *driver = (BlockHash) + *driver = (BlockChecksum) { .memContext = memContextCurrent(), .blockSize = blockSize, @@ -152,7 +152,7 @@ blockHashNew(const size_t blockSize, const size_t checksumSize) .list = lstNewP(checksumSize), }; - this = ioFilterNewP(BLOCK_HASH_FILTER_TYPE, driver, NULL, .in = blockHashProcess, .result = blockHashResult); + this = ioFilterNewP(BLOCK_CHECKSUM_FILTER_TYPE, driver, NULL, .in = blockChecksumProcess, .result = blockChecksumResult); } OBJ_NEW_END(); diff --git a/src/command/restore/blockHash.h b/src/command/restore/blockChecksum.h similarity index 80% rename from src/command/restore/blockHash.h rename to src/command/restore/blockChecksum.h index bd0a8691f..d34cc3697 100644 --- a/src/command/restore/blockHash.h +++ b/src/command/restore/blockChecksum.h @@ -4,8 +4,8 @@ Block Hash List Build a list of hashes based on a block size. This is used to compare the contents of a file to block map to determine what needs to be updated. ***********************************************************************************************************************************/ -#ifndef COMMAND_RESTORE_BLOCK_HASH_H -#define COMMAND_RESTORE_BLOCK_HASH_H +#ifndef COMMAND_RESTORE_BLOCK_CHECKSUM_H +#define COMMAND_RESTORE_BLOCK_CHECKSUM_H #include "common/io/filter/filter.h" #include "common/type/stringId.h" @@ -13,11 +13,11 @@ be updated. /*********************************************************************************************************************************** Filter type constant ***********************************************************************************************************************************/ -#define BLOCK_HASH_FILTER_TYPE STRID5("dlt-map", 0x402ddd1840) +#define BLOCK_CHECKSUM_FILTER_TYPE STRID5("blk-chk", 0x2d03dad820) /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ -FN_EXTERN IoFilter *blockHashNew(size_t blockSize, size_t checksumSize); +FN_EXTERN IoFilter *blockChecksumNew(size_t blockSize, size_t checksumSize); #endif diff --git a/src/command/restore/blockDelta.c b/src/command/restore/blockDelta.c index 080082188..68300fd2e 100644 --- a/src/command/restore/blockDelta.c +++ b/src/command/restore/blockDelta.c @@ -54,14 +54,14 @@ typedef struct BlockDeltaReference FN_EXTERN BlockDelta * blockDeltaNew( - const BlockMap *const blockMap, const size_t blockSize, const size_t checksumSize, const Buffer *const blockHash, + const BlockMap *const blockMap, const size_t blockSize, const size_t checksumSize, const Buffer *const blockChecksum, const CipherType cipherType, const String *const cipherPass, const CompressType compressType) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(BLOCK_MAP, blockMap); FUNCTION_TEST_PARAM(SIZE, blockSize); FUNCTION_TEST_PARAM(SIZE, checksumSize); - FUNCTION_TEST_PARAM(BUFFER, blockHash); + FUNCTION_TEST_PARAM(BUFFER, blockChecksum); FUNCTION_TEST_PARAM(STRING_ID, cipherType); FUNCTION_TEST_PARAM(STRING, cipherPass); FUNCTION_TEST_PARAM(ENUM, compressType); @@ -98,20 +98,20 @@ blockDeltaNew( MEM_CONTEXT_TEMP_BEGIN() { // Build list of references and for each reference the list of blocks for that reference - const unsigned int blockHashSize = - blockHash == NULL ? 0 : (unsigned int)(bufUsed(blockHash) / this->checksumSize); + const unsigned int blockChecksumSize = + blockChecksum == NULL ? 0 : (unsigned int)(bufUsed(blockChecksum) / this->checksumSize); List *const referenceList = lstNewP(sizeof(BlockDeltaReference), .comparator = lstComparatorUInt); for (unsigned int blockMapIdx = 0; blockMapIdx < blockMapSize(blockMap); blockMapIdx++) { const BlockMapItem *const blockMapItem = blockMapGet(blockMap, blockMapIdx); - // The block must be updated if it is beyond the blocks that exist in the block hash list or when the checksum - // stored in the repository is different from the block hash list - if (blockMapIdx >= blockHashSize || + // The block must be updated if it is beyond the blocks that exist in the block checksum list or when the checksum + // stored in the repository is different from the block checksum list + if (blockMapIdx >= blockChecksumSize || !bufEq( BUF(blockMapItem->checksum, this->checksumSize), - BUF(bufPtrConst(blockHash) + blockMapIdx * this->checksumSize, this->checksumSize))) + BUF(bufPtrConst(blockChecksum) + blockMapIdx * this->checksumSize, this->checksumSize))) { const unsigned int reference = blockMapItem->reference; BlockDeltaReference *const referenceData = lstFind(referenceList, &reference); diff --git a/src/command/restore/blockDelta.h b/src/command/restore/blockDelta.h index 687627e4e..5cacf03c2 100644 --- a/src/command/restore/blockDelta.h +++ b/src/command/restore/blockDelta.h @@ -1,8 +1,8 @@ /*********************************************************************************************************************************** Block Restore -Calculate and return the blocks required to restore a file using an optional block hash list. The block hash list is optional -because the file to restore may not exist so all the blocks will need to be restored. +Calculate and return the blocks required to restore a file using an optional block checksum list. The block checksum list is +optional because the file to restore may not exist so all the blocks will need to be restored. ***********************************************************************************************************************************/ #ifndef COMMAND_BACKUP_BLOCKDELTA_H #define COMMAND_BACKUP_BLOCKDELTA_H @@ -37,7 +37,7 @@ typedef struct BlockDeltaWrite Constructors ***********************************************************************************************************************************/ FN_EXTERN BlockDelta *blockDeltaNew( - const BlockMap *blockMap, size_t blockSize, size_t checksumSize, const Buffer *blockHash, CipherType cipherType, + const BlockMap *blockMap, size_t blockSize, size_t checksumSize, const Buffer *blockChecksum, CipherType cipherType, const String *cipherPass, const CompressType compressType); /*********************************************************************************************************************************** diff --git a/src/command/restore/file.c b/src/command/restore/file.c index 11dff72de..cf571a036 100644 --- a/src/command/restore/file.c +++ b/src/command/restore/file.c @@ -9,8 +9,8 @@ Restore File #include "command/backup/blockIncr.h" #include "command/backup/blockMap.h" +#include "command/restore/blockChecksum.h" #include "command/restore/blockDelta.h" -#include "command/restore/blockHash.h" #include "command/restore/file.h" #include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" @@ -125,12 +125,12 @@ restoreFile( read = storageReadIo(storageNewReadP(storagePg(), file->name)); ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1)); - // Generate block hash list if block incremental + // Generate block checksum list if block incremental if (file->blockIncrMapSize != 0) { ioFilterGroupAdd( ioReadFilterGroup(read), - blockHashNew(file->blockIncrSize, file->blockIncrChecksumSize)); + blockChecksumNew(file->blockIncrSize, file->blockIncrChecksumSize)); } ioReadDrain(read); @@ -142,9 +142,9 @@ restoreFile( file->checksum, pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE)))) { - // If the hash/size are now the same but the time is not, then set the time back to the backup - // time. This helps with unit testing, but also presents a pristine version of the database - // after restore. + // If the checksum/size are now the same but the time is not, then set the time back to the + // backup time. This helps with unit testing, but also presents a pristine version of the + // database after restore. if (info.timeModified != file->timeModified) { const struct utimbuf uTimeBuf = @@ -161,16 +161,16 @@ restoreFile( fileResult->result = restoreResultPreserve; } - // If block incremental and not preserving the file, store the block hash list for later use in + // If block incremental and not preserving the file, store the block checksum list for later use in // reconstructing the pg file if (file->blockIncrMapSize != 0 && fileResult->result != restoreResultPreserve) { - PackRead *const blockHashResult = ioFilterGroupResultP( - ioReadFilterGroup(read), BLOCK_HASH_FILTER_TYPE); + PackRead *const blockChecksumResult = ioFilterGroupResultP( + ioReadFilterGroup(read), BLOCK_CHECKSUM_FILTER_TYPE); MEM_CONTEXT_OBJ_BEGIN(fileList) { - file->blockHash = pckReadBinP(blockHashResult); + file->blockChecksum = pckReadBinP(blockChecksumResult); } MEM_CONTEXT_OBJ_END(); } @@ -284,7 +284,7 @@ restoreFile( StorageWrite *pgFileWrite = storageNewWriteP( storagePgWrite(), file->name, .modeFile = file->mode, .user = file->user, .group = file->group, .timeModified = file->timeModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true, - .noTruncate = file->blockHash != NULL); + .noTruncate = file->blockChecksum != NULL); // If block incremental file const Buffer *checksum = NULL; @@ -293,8 +293,8 @@ restoreFile( { ASSERT(referenceList != NULL); - // Read block map. This will be compared to the block hash list already created to determine which blocks - // need to be fetched from the repository. If we got here there must be at least one block to fetch. + // Read block map. This will be compared to the block checksum list already created to determine which + // blocks need to be fetched from the repository. If we got here there must be at least one block to fetch. const BlockMap *const blockMap = blockMapNewRead(storageReadIo(repoFileRead), file->blockIncrChecksumSize); // The repo file needs to be closed so that block lists can be read from the remote protocol @@ -305,7 +305,7 @@ restoreFile( // Apply delta to file BlockDelta *const blockDelta = blockDeltaNew( - blockMap, file->blockIncrSize, file->blockIncrChecksumSize, file->blockHash, + blockMap, file->blockIncrSize, file->blockIncrChecksumSize, file->blockChecksum, cipherPass == NULL ? cipherTypeNone : cipherTypeAes256Cbc, cipherPass, repoFileCompressType); for (unsigned int readIdx = 0; readIdx < blockDeltaReadSize(blockDelta); readIdx++) diff --git a/src/command/restore/file.h b/src/command/restore/file.h index d229a3c5f..78ec758ee 100644 --- a/src/command/restore/file.h +++ b/src/command/restore/file.h @@ -37,7 +37,7 @@ typedef struct RestoreFile size_t blockIncrSize; // Block incremental size (when map size > 0) size_t blockIncrChecksumSize; // Checksum size (when map size > 0) const String *manifestFile; // Manifest file - const Buffer *blockHash; // Hashes for block incremental restore, set in restoreFile() + const Buffer *blockChecksum; // Checksums for block incremental restore, set in restoreFile() } RestoreFile; typedef struct RestoreFileResult diff --git a/src/meson.build b/src/meson.build index 47f758b9d..a4aacdce8 100644 --- a/src/meson.build +++ b/src/meson.build @@ -146,8 +146,8 @@ src_pgbackrest = [ 'command/repo/ls.c', 'command/repo/put.c', 'command/repo/rm.c', + 'command/restore/blockChecksum.c', 'command/restore/blockDelta.c', - 'command/restore/blockHash.c', 'command/restore/file.c', 'command/restore/protocol.c', 'command/restore/restore.c', diff --git a/test/code-count/file-type.yaml b/test/code-count/file-type.yaml index d107f1495..2818d1842 100644 --- a/test/code-count/file-type.yaml +++ b/test/code-count/file-type.yaml @@ -575,11 +575,11 @@ src/command/repo/rm.h: class: core type: c/h -src/command/restore/blockHash.c: +src/command/restore/blockChecksum.c: class: core type: c -src/command/restore/blockHash.h: +src/command/restore/blockChecksum.h: class: core type: c/h diff --git a/test/define.yaml b/test/define.yaml index aff76233f..c94b2fef6 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -845,8 +845,8 @@ unit: total: 14 coverage: + - command/restore/blockChecksum - command/restore/blockDelta - - command/restore/blockHash - command/restore/file - command/restore/protocol - command/restore/restore diff --git a/test/src/module/command/restoreTest.c b/test/src/module/command/restoreTest.c index a415f762d..2a358d721 100644 --- a/test/src/module/command/restoreTest.c +++ b/test/src/module/command/restoreTest.c @@ -157,14 +157,14 @@ testRun(void) Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true); // ***************************************************************************************************************************** - if (testBegin("BlockHash")) + if (testBegin("BlockChecksum")) { // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("too large for one buffer"); Buffer *output = bufNew(0); IoWrite *write = ioBufferWriteNew(output); - ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3, 8)); + ioFilterGroupAdd(ioWriteFilterGroup(write), blockChecksumNew(3, 8)); ioWriteOpen(write); TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("ABCDEF")), "write"); @@ -172,11 +172,11 @@ testRun(void) TEST_RESULT_VOID(ioWriteClose(write), "close"); TEST_RESULT_STR_Z( - strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_HASH_FILTER_TYPE))), + strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_CHECKSUM_FILTER_TYPE))), "9e947f00ecd6acb2" "cb221327e5a387af" "9e947f00ecd6acb2", - "block hash list"); + "block checksum list"); ioWriteFree(write); @@ -185,7 +185,7 @@ testRun(void) output = bufNew(0); write = ioBufferWriteNew(output); - ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3, 8)); + ioFilterGroupAdd(ioWriteFilterGroup(write), blockChecksumNew(3, 8)); ioWriteOpen(write); TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("DE")), "write"); @@ -196,12 +196,12 @@ testRun(void) TEST_RESULT_VOID(ioWriteClose(write), "close"); TEST_RESULT_STR_Z( - strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_HASH_FILTER_TYPE))), + strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_CHECKSUM_FILTER_TYPE))), "cb221327e5a387af" "9e947f00ecd6acb2" "9e947f00ecd6acb2" "92c3453969207870", - "block hash list"); + "block checksum list"); ioWriteFree(write); }