db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-02-21 19:48:29 +02:00
Code Issues Releases Activity

Rename DeltaMap to BlockHash.

Browse Source 
This more accurately describes what the object does.
This commit is contained in:
David Steele 2023-02-13 09:17:30 +07:00
parent 779efe0d7a
commit dffc933384
10 changed files with 59 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/xml/release.xml
View File

@ -22,6 +22,7 @@
<commit subject="Improve IoChunkedRead end-of-file handling."/>
<commit subject="Remove parameter list from deltaMapNew()."/>
<commit subject="Consistently declare block incremental size as size_t."/>
<commit subject="Rename DeltaMap to BlockHash."/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>

2
src/Makefile.in
View File

@ -82,7 +82,7 @@ SRCS = \
command/repo/ls.c \
command/repo/put.c \
command/repo/rm.c \
command/restore/deltaMap.c \
command/restore/blockHash.c \
command/restore/file.c \
command/restore/protocol.c \
command/restore/restore.c \

40
src/command/restore/deltaMap.c → src/command/restore/blockHash.c
View File

@ -3,7 +3,7 @@ Restore Delta Map
***********************************************************************************************************************************/
#include "build.auto.h"
#include "command/restore/deltaMap.h"
#include "command/restore/blockHash.h"
#include "common/crypto/common.h"
#include "common/crypto/hash.h"
#include "common/debug.h"
@ -13,34 +13,34 @@ Restore Delta Map
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct DeltaMap
typedef struct BlockHash
{
MemContext *memContext; // Mem context of filter
size_t blockSize; // Block size for checksums
size_t blockCurrent; // Size of current block
IoFilter *hash; // Hash of current block
List *list; // List if hashes
} DeltaMap;
List *list; // List of hashes
} BlockHash;
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
#define FUNCTION_LOG_DELTA_MAP_TYPE \
DeltaMap *
#define FUNCTION_LOG_DELTA_MAP_FORMAT(value, buffer, bufferSize) \
objNameToLog(value, "DeltaMap", buffer, bufferSize)
#define FUNCTION_LOG_BLOCK_HASH_TYPE \
BlockHash *
#define FUNCTION_LOG_BLOCK_HASH_FORMAT(value, buffer, bufferSize) \
objNameToLog(value, "BlockHash", buffer, bufferSize)
/***********************************************************************************************************************************
Generate delta map
Generate block hash list
***********************************************************************************************************************************/
static void
deltaMapProcess(THIS_VOID, const Buffer *const input)
blockHashProcess(THIS_VOID, const Buffer *const input)
{
THIS(DeltaMap);
THIS(BlockHash);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(DELTA_MAP, this);
FUNCTION_LOG_PARAM(BLOCK_HASH, this);
FUNCTION_LOG_PARAM(BUFFER, input);
FUNCTION_LOG_END();
@ -95,12 +95,12 @@ deltaMapProcess(THIS_VOID, const Buffer *const input)
Get a binary representation of the hash list
***********************************************************************************************************************************/
static Pack *
deltaMapResult(THIS_VOID)
blockHashResult(THIS_VOID)
{
THIS(DeltaMap);
THIS(BlockHash);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(DELTA_MAP, this);
FUNCTION_LOG_PARAM(BLOCK_HASH, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
@ -127,7 +127,7 @@ deltaMapResult(THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
deltaMapNew(const size_t blockSize)
blockHashNew(const size_t blockSize)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(SIZE, blockSize);
@ -138,18 +138,18 @@ deltaMapNew(const size_t blockSize)
// Allocate memory to hold process state
IoFilter *this = NULL;
OBJ_NEW_BEGIN(DeltaMap, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
OBJ_NEW_BEGIN(BlockHash, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
{
DeltaMap *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), IoFilter::DeltaMap);
BlockHash *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), IoFilter::BlockHash);
*driver = (DeltaMap)
*driver = (BlockHash)
{
.memContext = memContextCurrent(),
.blockSize = blockSize,
.list = lstNewP(HASH_TYPE_SHA1_SIZE),
};
this = ioFilterNewP(DELTA_MAP_FILTER_TYPE, driver, NULL, .in = deltaMapProcess, .result = deltaMapResult);
this = ioFilterNewP(BLOCK_HASH_FILTER_TYPE, driver, NULL, .in = blockHashProcess, .result = blockHashResult);
}
OBJ_NEW_END();

10
src/command/restore/deltaMap.h → src/command/restore/blockHash.h
View File

@ -1,11 +1,11 @@
/***********************************************************************************************************************************
Restore Delta Map
Block Hash List
Build a list of hashes based on a block size. This is used to compare the contents of a file to block map to determine what needs to
be updated.
***********************************************************************************************************************************/
#ifndef COMMAND_RESTORE_DELTA_MAP_H
#define COMMAND_RESTORE_DELTA_MAP_H
#ifndef COMMAND_RESTORE_BLOCK_HASH_H
#define COMMAND_RESTORE_BLOCK_HASH_H
#include "common/io/filter/filter.h"
#include "common/type/stringId.h"
@ -13,11 +13,11 @@ be updated.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define DELTA_MAP_FILTER_TYPE STRID5("dlt-map", 0x402ddd1840)
#define BLOCK_HASH_FILTER_TYPE STRID5("dlt-map", 0x402ddd1840)
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN IoFilter *deltaMapNew(size_t blockSize);
FN_EXTERN IoFilter *blockHashNew(size_t blockSize);
#endif

40
src/command/restore/file.c
View File

@ -8,7 +8,7 @@ Restore File
#include <utime.h>
#include "command/backup/blockMap.h"
#include "command/restore/deltaMap.h"
#include "command/restore/blockHash.h"
#include "command/restore/file.h"
#include "common/crypto/cipherBlock.h"
#include "common/crypto/hash.h"
@ -122,9 +122,9 @@ restoreFile(
read = storageReadIo(storageNewReadP(storagePg(), file->name));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1));
// Generate delta map if block incremental
// Generate block hash list if block incremental
if (file->blockIncrMapSize != 0)
ioFilterGroupAdd(ioReadFilterGroup(read), deltaMapNew(file->blockIncrSize));
ioFilterGroupAdd(ioReadFilterGroup(read), blockHashNew(file->blockIncrSize));
ioReadDrain(read);
}
@ -154,16 +154,16 @@ restoreFile(
fileResult->result = restoreResultPreserve;
}
// If block incremental and not preserving the file, store the delta map for later use in
// If block incremental and not preserving the file, store the block hash list for later use in
// reconstructing the pg file
if (file->blockIncrMapSize != 0 && fileResult->result != restoreResultPreserve)
{
PackRead *const deltaMapResult = ioFilterGroupResultP(
ioReadFilterGroup(read), DELTA_MAP_FILTER_TYPE);
PackRead *const blockHashResult = ioFilterGroupResultP(
ioReadFilterGroup(read), BLOCK_HASH_FILTER_TYPE);
MEM_CONTEXT_OBJ_BEGIN(fileList)
{
file->deltaMap = pckReadBinP(deltaMapResult);
file->blockHash = pckReadBinP(blockHashResult);
}
MEM_CONTEXT_OBJ_END();
}
@ -277,7 +277,7 @@ restoreFile(
StorageWrite *pgFileWrite = storageNewWriteP(
storagePgWrite(), file->name, .modeFile = file->mode, .user = file->user, .group = file->group,
.timeModified = file->timeModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true,
.noTruncate = file->deltaMap != NULL);
.noTruncate = file->blockHash != NULL);
// If block incremental file
const Buffer *checksum = NULL;
@ -286,17 +286,17 @@ restoreFile(
{
ASSERT(referenceList != NULL);
// Read block map. This will be compared to the delta map already created to determine which blocks need to
// be fetched from the repository. If we got here there must be at least one block to fetch.
// Read block map. This will be compared to the block hash list already created to determine which blocks
// need to be fetched from the repository. If we got here there must be at least one block to fetch.
const BlockMap *const blockMap = blockMapNewRead(storageReadIo(repoFileRead));
// The repo file needs to be closed so that block lists can be read from the remote protocol
ioReadClose(storageReadIo(repoFileRead));
// Size of delta map. If there is no delta map because the pg file does not exist then set to zero, which
// will force all blocks to be updated.
const unsigned int deltaMapSize =
file->deltaMap == NULL ? 0 : (unsigned int)(bufUsed(file->deltaMap) / HASH_TYPE_SHA1_SIZE);
// Size of block hash list. If there is no block hash list because the pg file does not exist then set to
// zero, which will force all blocks to be updated.
const unsigned int blockHashSize =
file->blockHash == NULL ? 0 : (unsigned int)(bufUsed(file->blockHash) / HASH_TYPE_SHA1_SIZE);
// Find and write updated blocks
bool updateFound = false; // Is there a block list to be updated?
@ -311,12 +311,12 @@ restoreFile(
{
const BlockMapItem *const blockMapItem = blockMapGet(blockMap, blockMapIdx);
// The block must be updated if it beyond the blocks that exist in the delta map or when the checksum
// stored in the repository is different from the delta map
if (blockMapIdx >= deltaMapSize ||
// The block must be updated if it beyond the blocks that exist in the block hash list or when the
// checksum stored in the repository is different from the block hash list
if (blockMapIdx >= blockHashSize ||
!bufEq(
BUF(blockMapItem->checksum, HASH_TYPE_SHA1_SIZE),
BUF(bufPtrConst(file->deltaMap) + blockMapIdx * HASH_TYPE_SHA1_SIZE, HASH_TYPE_SHA1_SIZE)))
BUF(bufPtrConst(file->blockHash) + blockMapIdx * HASH_TYPE_SHA1_SIZE, HASH_TYPE_SHA1_SIZE)))
{
// If no block list is currently being built then start a new one
if (!updateFound)
@ -343,11 +343,11 @@ restoreFile(
// Similar to the check above, but also make sure the reference is the same. For blocks to be
// in a common list they must be contiguous and from the same reference.
if (blockMapItem->reference == blockMapItemNext->reference &&
(blockMapIdx + 1 >= deltaMapSize ||
(blockMapIdx + 1 >= blockHashSize ||
!bufEq(
BUF(blockMapItemNext->checksum, HASH_TYPE_SHA1_SIZE),
BUF(
bufPtrConst(file->deltaMap) + (blockMapIdx + 1) * HASH_TYPE_SHA1_SIZE,
bufPtrConst(file->blockHash) + (blockMapIdx + 1) * HASH_TYPE_SHA1_SIZE,
HASH_TYPE_SHA1_SIZE))))
{
continue;

2
src/command/restore/file.h
View File

@ -36,7 +36,7 @@ typedef struct RestoreFile
uint64_t blockIncrMapSize; // Block incremental map size (0 if not incremental)
size_t blockIncrSize; // Block incremental size (when map size > 0)
const String *manifestFile; // Manifest file
const Buffer *deltaMap; // Delta for block incremental restore, set in restoreFile()
const Buffer *blockHash; // Hashes for block incremental restore, set in restoreFile()
} RestoreFile;
typedef struct RestoreFileResult

2
src/meson.build
View File

@ -146,7 +146,7 @@ src_pgbackrest = [
'command/repo/ls.c',
'command/repo/put.c',
'command/repo/rm.c',
'command/restore/deltaMap.c',
'command/restore/blockHash.c',
'command/restore/file.c',
'command/restore/protocol.c',
'command/restore/restore.c',

4
test/code-count/file-type.yaml
View File

@ -575,11 +575,11 @@ src/command/repo/rm.h:
class: core
type: c/h
src/command/restore/deltaMap.c:
src/command/restore/blockHash.c:
class: core
type: c
src/command/restore/deltaMap.h:
src/command/restore/blockHash.h:
class: core
type: c/h

2
test/define.yaml
View File

@ -843,7 +843,7 @@ unit:
total: 14
coverage:
- command/restore/deltaMap
- command/restore/blockHash
- command/restore/file
- command/restore/protocol
- command/restore/restore

14
test/src/module/command/restoreTest.c
View File

@ -157,14 +157,14 @@ testRun(void)
Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true);
// *****************************************************************************************************************************
if (testBegin("DeltaMap"))
if (testBegin("BlockHash"))
{
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("too large for one buffer");
Buffer *output = bufNew(0);
IoWrite *write = ioBufferWriteNew(output);
ioFilterGroupAdd(ioWriteFilterGroup(write), deltaMapNew(3));
ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3));
ioWriteOpen(write);
TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("ABCDEF")), "write");
@ -172,18 +172,18 @@ testRun(void)
TEST_RESULT_VOID(ioWriteClose(write), "close");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), DELTA_MAP_FILTER_TYPE))),
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_HASH_FILTER_TYPE))),
"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8"
"6dae29c06c5f04601445c493156d10fe1be23b6d"
"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8",
"delta map");
"block hash list");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("buffer smaller than block and remainder");
output = bufNew(0);
write = ioBufferWriteNew(output);
ioFilterGroupAdd(ioWriteFilterGroup(write), deltaMapNew(3));
ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3));
ioWriteOpen(write);
TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("DE")), "write");
@ -194,12 +194,12 @@ testRun(void)
TEST_RESULT_VOID(ioWriteClose(write), "close");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), DELTA_MAP_FILTER_TYPE))),
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_HASH_FILTER_TYPE))),
"6dae29c06c5f04601445c493156d10fe1be23b6d"
"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8"
"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8"
"c032adc1ff629c9b66f22749ad667e6beadf144b",
"delta map");
"block hash list");
}
// *****************************************************************************************************************************