db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Rename BlockHash to BlockChecksum.

Browse Source 
Checksum is the generally used terminology in the code base, even when a hash is being used as a checksum.
This commit is contained in:
David Steele 2023-03-09 11:04:03 +07:00
parent 6252c0e448
commit 1119a53539
12 changed files with 82 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/xml/release.xml
View File

@ -50,6 +50,7 @@
</commit>
<commit subject="Use xxHash instead of SHA-1 for block incremental checksums."/>
<commit subject="Exclude backup set size from info for block incremental backups."/>
<commit subject="Rename BlockHash to BlockChecksum."/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>

2
src/Makefile.in
View File

@ -82,8 +82,8 @@ SRCS = \
command/repo/ls.c \
command/repo/put.c \
command/repo/rm.c \
command/restore/blockChecksum.c \
command/restore/blockDelta.c \
command/restore/blockHash.c \
command/restore/file.c \
command/restore/protocol.c \
command/restore/restore.c \

78
src/command/restore/blockHash.c → src/command/restore/blockChecksum.c
View File

@ -1,9 +1,9 @@
/***********************************************************************************************************************************
Restore Delta Map
Block Hash List
***********************************************************************************************************************************/
#include "build.auto.h"
#include "command/restore/blockHash.h"
#include "command/restore/blockChecksum.h"
#include "common/crypto/common.h"
#include "common/crypto/xxhash.h"
#include "common/debug.h"
@ -13,35 +13,35 @@ Restore Delta Map
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct BlockHash
typedef struct BlockChecksum
{
MemContext *memContext; // Mem context of filter
size_t blockSize; // Block size for checksums
size_t checksumSize; // Checksum size
size_t blockCurrent; // Size of current block
IoFilter *hash; // Hash of current block
List *list; // List of hashes
} BlockHash;
IoFilter *checksum; // Checksum of current block
List *list; // List of checksums
} BlockChecksum;
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
#define FUNCTION_LOG_BLOCK_HASH_TYPE \
BlockHash *
#define FUNCTION_LOG_BLOCK_HASH_FORMAT(value, buffer, bufferSize) \
objNameToLog(value, "BlockHash", buffer, bufferSize)
#define FUNCTION_LOG_BLOCK_CHECKSUM_TYPE \
BlockChecksum *
#define FUNCTION_LOG_BLOCK_CHECKSUM_FORMAT(value, buffer, bufferSize) \
objNameToLog(value, "BlockChecksum", buffer, bufferSize)
/***********************************************************************************************************************************
Generate block hash list
Generate block checksum list
***********************************************************************************************************************************/
static void
blockHashProcess(THIS_VOID, const Buffer *const input)
blockChecksumProcess(THIS_VOID, const Buffer *const input)
{
THIS(BlockHash);
THIS(BlockChecksum);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BLOCK_HASH, this);
FUNCTION_LOG_PARAM(BLOCK_CHECKSUM, this);
FUNCTION_LOG_PARAM(BUFFER, input);
FUNCTION_LOG_END();
@ -53,37 +53,37 @@ blockHashProcess(THIS_VOID, const Buffer *const input)
// Loop until input is consumed
while (inputOffset != bufUsed(input))
{
// Create hash object if needed
if (this->hash == NULL)
// Create checksum object if needed
if (this->checksum == NULL)
{
MEM_CONTEXT_BEGIN(this->memContext)
{
this->hash = xxHashNew(this->checksumSize);
this->checksum = xxHashNew(this->checksumSize);
this->blockCurrent = 0;
}
MEM_CONTEXT_END();
}
// Calculate how much data to hash and perform hash
// Calculate how much data to checksum and perform checksum
const size_t blockRemains = this->blockSize - this->blockCurrent;
const size_t inputRemains = bufUsed(input) - inputOffset;
const size_t blockHash = blockRemains < inputRemains ? blockRemains : inputRemains;
const size_t blockChecksumSize = blockRemains < inputRemains ? blockRemains : inputRemains;
ioFilterProcessIn(this->hash, BUF(bufPtrConst(input) + inputOffset, blockHash));
ioFilterProcessIn(this->checksum, BUF(bufPtrConst(input) + inputOffset, blockChecksumSize));
// Update amount of data hashed
inputOffset += blockHash;
this->blockCurrent += blockHash;
// Update amount of data checksummed
inputOffset += blockChecksumSize;
this->blockCurrent += blockChecksumSize;
// If the block size has been reached then output the hash
// If the block size has been reached then output the checksum
if (this->blockCurrent == this->blockSize)
{
MEM_CONTEXT_TEMP_BEGIN()
{
lstAdd(this->list, bufPtrConst(pckReadBinP(pckReadNew(ioFilterResult(this->hash)))));
lstAdd(this->list, bufPtrConst(pckReadBinP(pckReadNew(ioFilterResult(this->checksum)))));
ioFilterFree(this->hash);
this->hash = NULL;
ioFilterFree(this->checksum);
this->checksum = NULL;
}
MEM_CONTEXT_TEMP_END();
}
@ -93,15 +93,15 @@ blockHashProcess(THIS_VOID, const Buffer *const input)
}
/***********************************************************************************************************************************
Get a binary representation of the hash list
Get a binary representation of the checksum list
***********************************************************************************************************************************/
static Pack *
blockHashResult(THIS_VOID)
blockChecksumResult(THIS_VOID)
{
THIS(BlockHash);
THIS(BlockChecksum);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BLOCK_HASH, this);
FUNCTION_LOG_PARAM(BLOCK_CHECKSUM, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
@ -112,9 +112,9 @@ blockHashResult(THIS_VOID)
{
PackWrite *const packWrite = pckWriteNewP();
// If there is a remainder in the hash
if (this->hash)
lstAdd(this->list, bufPtrConst(pckReadBinP(pckReadNew(ioFilterResult(this->hash)))));
// If there is a remainder in the checksum
if (this->checksum)
lstAdd(this->list, bufPtrConst(pckReadBinP(pckReadNew(ioFilterResult(this->checksum)))));
pckWriteBinP(packWrite, BUF(lstGet(this->list, 0), lstSize(this->list) * this->checksumSize));
pckWriteEndP(packWrite);
@ -128,7 +128,7 @@ blockHashResult(THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
blockHashNew(const size_t blockSize, const size_t checksumSize)
blockChecksumNew(const size_t blockSize, const size_t checksumSize)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(SIZE, blockSize);
@ -140,11 +140,11 @@ blockHashNew(const size_t blockSize, const size_t checksumSize)
// Allocate memory to hold process state
IoFilter *this = NULL;
OBJ_NEW_BEGIN(BlockHash, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
OBJ_NEW_BEGIN(BlockChecksum, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
{
BlockHash *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), IoFilter::BlockHash);
BlockChecksum *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), IoFilter::BlockChecksum);
*driver = (BlockHash)
*driver = (BlockChecksum)
{
.memContext = memContextCurrent(),
.blockSize = blockSize,
@ -152,7 +152,7 @@ blockHashNew(const size_t blockSize, const size_t checksumSize)
.list = lstNewP(checksumSize),
};
this = ioFilterNewP(BLOCK_HASH_FILTER_TYPE, driver, NULL, .in = blockHashProcess, .result = blockHashResult);
this = ioFilterNewP(BLOCK_CHECKSUM_FILTER_TYPE, driver, NULL, .in = blockChecksumProcess, .result = blockChecksumResult);
}
OBJ_NEW_END();

8
src/command/restore/blockHash.h → src/command/restore/blockChecksum.h
View File

@ -4,8 +4,8 @@ Block Hash List
Build a list of hashes based on a block size. This is used to compare the contents of a file to block map to determine what needs to
be updated.
***********************************************************************************************************************************/
#ifndef COMMAND_RESTORE_BLOCK_HASH_H
#define COMMAND_RESTORE_BLOCK_HASH_H
#ifndef COMMAND_RESTORE_BLOCK_CHECKSUM_H
#define COMMAND_RESTORE_BLOCK_CHECKSUM_H
#include "common/io/filter/filter.h"
#include "common/type/stringId.h"
@ -13,11 +13,11 @@ be updated.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define BLOCK_HASH_FILTER_TYPE STRID5("dlt-map", 0x402ddd1840)
#define BLOCK_CHECKSUM_FILTER_TYPE STRID5("blk-chk", 0x2d03dad820)
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN IoFilter *blockHashNew(size_t blockSize, size_t checksumSize);
FN_EXTERN IoFilter *blockChecksumNew(size_t blockSize, size_t checksumSize);
#endif

16
src/command/restore/blockDelta.c
View File

@ -54,14 +54,14 @@ typedef struct BlockDeltaReference
FN_EXTERN BlockDelta *
blockDeltaNew(
const BlockMap *const blockMap, const size_t blockSize, const size_t checksumSize, const Buffer *const blockHash,
const BlockMap *const blockMap, const size_t blockSize, const size_t checksumSize, const Buffer *const blockChecksum,
const CipherType cipherType, const String *const cipherPass, const CompressType compressType)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BLOCK_MAP, blockMap);
FUNCTION_TEST_PARAM(SIZE, blockSize);
FUNCTION_TEST_PARAM(SIZE, checksumSize);
FUNCTION_TEST_PARAM(BUFFER, blockHash);
FUNCTION_TEST_PARAM(BUFFER, blockChecksum);
FUNCTION_TEST_PARAM(STRING_ID, cipherType);
FUNCTION_TEST_PARAM(STRING, cipherPass);
FUNCTION_TEST_PARAM(ENUM, compressType);
@ -98,20 +98,20 @@ blockDeltaNew(
MEM_CONTEXT_TEMP_BEGIN()
{
// Build list of references and for each reference the list of blocks for that reference
const unsigned int blockHashSize =
blockHash == NULL ? 0 : (unsigned int)(bufUsed(blockHash) / this->checksumSize);
const unsigned int blockChecksumSize =
blockChecksum == NULL ? 0 : (unsigned int)(bufUsed(blockChecksum) / this->checksumSize);
List *const referenceList = lstNewP(sizeof(BlockDeltaReference), .comparator = lstComparatorUInt);
for (unsigned int blockMapIdx = 0; blockMapIdx < blockMapSize(blockMap); blockMapIdx++)
{
const BlockMapItem *const blockMapItem = blockMapGet(blockMap, blockMapIdx);
// The block must be updated if it is beyond the blocks that exist in the block hash list or when the checksum
// stored in the repository is different from the block hash list
if (blockMapIdx >= blockHashSize ||
// The block must be updated if it is beyond the blocks that exist in the block checksum list or when the checksum
// stored in the repository is different from the block checksum list
if (blockMapIdx >= blockChecksumSize ||
!bufEq(
BUF(blockMapItem->checksum, this->checksumSize),
BUF(bufPtrConst(blockHash) + blockMapIdx * this->checksumSize, this->checksumSize)))
BUF(bufPtrConst(blockChecksum) + blockMapIdx * this->checksumSize, this->checksumSize)))
{
const unsigned int reference = blockMapItem->reference;
BlockDeltaReference *const referenceData = lstFind(referenceList, &reference);

6
src/command/restore/blockDelta.h
View File

@ -1,8 +1,8 @@
/***********************************************************************************************************************************
Block Restore
Calculate and return the blocks required to restore a file using an optional block hash list. The block hash list is optional
because the file to restore may not exist so all the blocks will need to be restored.
Calculate and return the blocks required to restore a file using an optional block checksum list. The block checksum list is
optional because the file to restore may not exist so all the blocks will need to be restored.
***********************************************************************************************************************************/
#ifndef COMMAND_BACKUP_BLOCKDELTA_H
#define COMMAND_BACKUP_BLOCKDELTA_H
@ -37,7 +37,7 @@ typedef struct BlockDeltaWrite
Constructors
***********************************************************************************************************************************/
FN_EXTERN BlockDelta *blockDeltaNew(
const BlockMap *blockMap, size_t blockSize, size_t checksumSize, const Buffer *blockHash, CipherType cipherType,
const BlockMap *blockMap, size_t blockSize, size_t checksumSize, const Buffer *blockChecksum, CipherType cipherType,
const String *cipherPass, const CompressType compressType);
/***********************************************************************************************************************************

28
src/command/restore/file.c
View File

@ -9,8 +9,8 @@ Restore File
#include "command/backup/blockIncr.h"
#include "command/backup/blockMap.h"
#include "command/restore/blockChecksum.h"
#include "command/restore/blockDelta.h"
#include "command/restore/blockHash.h"
#include "command/restore/file.h"
#include "common/crypto/cipherBlock.h"
#include "common/crypto/hash.h"
@ -125,12 +125,12 @@ restoreFile(
read = storageReadIo(storageNewReadP(storagePg(), file->name));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1));
// Generate block hash list if block incremental
// Generate block checksum list if block incremental
if (file->blockIncrMapSize != 0)
{
ioFilterGroupAdd(
ioReadFilterGroup(read),
blockHashNew(file->blockIncrSize, file->blockIncrChecksumSize));
blockChecksumNew(file->blockIncrSize, file->blockIncrChecksumSize));
}
ioReadDrain(read);
@ -142,9 +142,9 @@ restoreFile(
file->checksum,
pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE))))
{
// If the hash/size are now the same but the time is not, then set the time back to the backup
// time. This helps with unit testing, but also presents a pristine version of the database
// after restore.
// If the checksum/size are now the same but the time is not, then set the time back to the
// backup time. This helps with unit testing, but also presents a pristine version of the
// database after restore.
if (info.timeModified != file->timeModified)
{
const struct utimbuf uTimeBuf =
@ -161,16 +161,16 @@ restoreFile(
fileResult->result = restoreResultPreserve;
}
// If block incremental and not preserving the file, store the block hash list for later use in
// If block incremental and not preserving the file, store the block checksum list for later use in
// reconstructing the pg file
if (file->blockIncrMapSize != 0 && fileResult->result != restoreResultPreserve)
{
PackRead *const blockHashResult = ioFilterGroupResultP(
ioReadFilterGroup(read), BLOCK_HASH_FILTER_TYPE);
PackRead *const blockChecksumResult = ioFilterGroupResultP(
ioReadFilterGroup(read), BLOCK_CHECKSUM_FILTER_TYPE);
MEM_CONTEXT_OBJ_BEGIN(fileList)
{
file->blockHash = pckReadBinP(blockHashResult);
file->blockChecksum = pckReadBinP(blockChecksumResult);
}
MEM_CONTEXT_OBJ_END();
}
@ -284,7 +284,7 @@ restoreFile(
StorageWrite *pgFileWrite = storageNewWriteP(
storagePgWrite(), file->name, .modeFile = file->mode, .user = file->user, .group = file->group,
.timeModified = file->timeModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true,
.noTruncate = file->blockHash != NULL);
.noTruncate = file->blockChecksum != NULL);
// If block incremental file
const Buffer *checksum = NULL;
@ -293,8 +293,8 @@ restoreFile(
{
ASSERT(referenceList != NULL);
// Read block map. This will be compared to the block hash list already created to determine which blocks
// need to be fetched from the repository. If we got here there must be at least one block to fetch.
// Read block map. This will be compared to the block checksum list already created to determine which
// blocks need to be fetched from the repository. If we got here there must be at least one block to fetch.
const BlockMap *const blockMap = blockMapNewRead(storageReadIo(repoFileRead), file->blockIncrChecksumSize);
// The repo file needs to be closed so that block lists can be read from the remote protocol
@ -305,7 +305,7 @@ restoreFile(
// Apply delta to file
BlockDelta *const blockDelta = blockDeltaNew(
blockMap, file->blockIncrSize, file->blockIncrChecksumSize, file->blockHash,
blockMap, file->blockIncrSize, file->blockIncrChecksumSize, file->blockChecksum,
cipherPass == NULL ? cipherTypeNone : cipherTypeAes256Cbc, cipherPass, repoFileCompressType);
for (unsigned int readIdx = 0; readIdx < blockDeltaReadSize(blockDelta); readIdx++)

2
src/command/restore/file.h
View File

@ -37,7 +37,7 @@ typedef struct RestoreFile
size_t blockIncrSize; // Block incremental size (when map size > 0)
size_t blockIncrChecksumSize; // Checksum size (when map size > 0)
const String *manifestFile; // Manifest file
const Buffer *blockHash; // Hashes for block incremental restore, set in restoreFile()
const Buffer *blockChecksum; // Checksums for block incremental restore, set in restoreFile()
} RestoreFile;
typedef struct RestoreFileResult

2
src/meson.build
View File

@ -146,8 +146,8 @@ src_pgbackrest = [
'command/repo/ls.c',
'command/repo/put.c',
'command/repo/rm.c',
'command/restore/blockChecksum.c',
'command/restore/blockDelta.c',
'command/restore/blockHash.c',
'command/restore/file.c',
'command/restore/protocol.c',
'command/restore/restore.c',

4
test/code-count/file-type.yaml
View File

@ -575,11 +575,11 @@ src/command/repo/rm.h:
class: core
type: c/h
src/command/restore/blockHash.c:
src/command/restore/blockChecksum.c:
class: core
type: c
src/command/restore/blockHash.h:
src/command/restore/blockChecksum.h:
class: core
type: c/h

2
test/define.yaml
View File

@ -845,8 +845,8 @@ unit:
total: 14
coverage:
- command/restore/blockChecksum
- command/restore/blockDelta
- command/restore/blockHash
- command/restore/file
- command/restore/protocol
- command/restore/restore

14
test/src/module/command/restoreTest.c
View File

@ -157,14 +157,14 @@ testRun(void)
Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true);
// *****************************************************************************************************************************
if (testBegin("BlockHash"))
if (testBegin("BlockChecksum"))
{
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("too large for one buffer");
Buffer *output = bufNew(0);
IoWrite *write = ioBufferWriteNew(output);
ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3, 8));
ioFilterGroupAdd(ioWriteFilterGroup(write), blockChecksumNew(3, 8));
ioWriteOpen(write);
TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("ABCDEF")), "write");
@ -172,11 +172,11 @@ testRun(void)
TEST_RESULT_VOID(ioWriteClose(write), "close");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_HASH_FILTER_TYPE))),
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_CHECKSUM_FILTER_TYPE))),
"9e947f00ecd6acb2"
"cb221327e5a387af"
"9e947f00ecd6acb2",
"block hash list");
"block checksum list");
ioWriteFree(write);
@ -185,7 +185,7 @@ testRun(void)
output = bufNew(0);
write = ioBufferWriteNew(output);
ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3, 8));
ioFilterGroupAdd(ioWriteFilterGroup(write), blockChecksumNew(3, 8));
ioWriteOpen(write);
TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("DE")), "write");
@ -196,12 +196,12 @@ testRun(void)
TEST_RESULT_VOID(ioWriteClose(write), "close");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_HASH_FILTER_TYPE))),
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_CHECKSUM_FILTER_TYPE))),
"cb221327e5a387af"
"9e947f00ecd6acb2"
"9e947f00ecd6acb2"
"92c3453969207870",
"block hash list");
"block checksum list");
ioWriteFree(write);
}