Use xxHash instead of SHA-1 for block incremental checksums.

xxHash is significantly faster than SHA-1 so this helps reduce the overhead of the feature.

A variable number of bytes are used from the xxHash depending on the block size with a minimum of six bytes for the smallest block size. This keeps the maps smaller while still providing enough bits to detect block changes.

doc/xml/release.xml

@@ -48,10 +48,13 @@
                         <commit subject="Block-level incremental backup super blocks.">
                             <github-pull-request id="2011"/>
                         </commit>
+                        <commit subject="Use xxHash instead of SHA-1 for block incremental checksums."/>
 
                         <release-item-contributor-list>
                             <release-item-contributor id="david.steele"/>
                             <release-item-reviewer id="john.morris"/>
+                            <release-item-reviewer id="stephen.frost"/>
+                            <release-item-reviewer id="stefan.fercot"/>
                         </release-item-contributor-list>
 
                         <p>Block-level incremental backup (BETA).</p>

src/Makefile.in

@@ -111,6 +111,7 @@ SRCS = \
 	common/crypto/cipherBlock.c \
 	common/crypto/common.c \
 	common/crypto/hash.c \
+	common/crypto/xxhash.c \
 	common/exec.c \
 	common/fork.c \
 	common/ini.c \

src/build/config/config.yaml

@@ -1866,6 +1866,9 @@ option:
   repo-block-age-map:
     inherit: repo-block-size-map
 
+  repo-block-checksum-size-map:
+    inherit: repo-block-size-map
+
   repo-block-size-super:
     section: global
     group: repo

src/build/help/help.xml

@@ -532,6 +532,16 @@
                         <example>7=2</example>
                     </config-key>
 
+                    <config-key id="repo-block-checksum-size-map" name="Block Incremental Checksum Size Map">
+                        <summary>Block incremental checksum size map.</summary>
+
+                        <text>
+                            <p>Map block size to checksum size. Smaller checksums save space in the map but may not be able to reliably detect changes in the block.</p>
+                       </text>
+
+                        <example>7=2</example>
+                    </config-key>
+
                     <config-key id="repo-block-size-map" name="Block Incremental Size Map">
                         <summary>Block incremental size map.</summary>
 

src/command/backup/backup.c

@@ -294,6 +294,17 @@ static const ManifestBlockIncrAgeMap manifestBlockIncrAgeMapDefault[] =
     {.fileAge = 7 * SEC_PER_DAY, .blockMultiplier = 2},
 };
 
+// Checksum size map
+static const ManifestBlockIncrChecksumSizeMap manifestBlockIncrChecksumSizeMapDefault[] =
+{
+    {.blockSize = 4 * 1024 * 1024, .checksumSize = BLOCK_INCR_CHECKSUM_SIZE_MIN + 6},
+    {.blockSize = 2 * 1024 * 1024, .checksumSize = BLOCK_INCR_CHECKSUM_SIZE_MIN + 5},
+    {.blockSize = 1024 * 1024, .checksumSize = BLOCK_INCR_CHECKSUM_SIZE_MIN + 4},
+    {.blockSize = 512 * 1024, .checksumSize = BLOCK_INCR_CHECKSUM_SIZE_MIN + 3},
+    {.blockSize = 128 * 1024, .checksumSize = BLOCK_INCR_CHECKSUM_SIZE_MIN + 2},
+    {.blockSize = 32 * 1024, .checksumSize = BLOCK_INCR_CHECKSUM_SIZE_MIN + 1},
+};
+
 // All maps
 static const ManifestBlockIncrMap manifestBlockIncrMap =
 {
@@ -301,6 +312,8 @@ static const ManifestBlockIncrMap manifestBlockIncrMap =
     .sizeMapSize = LENGTH_OF(manifestBlockIncrSizeMapDefault),
     .ageMap = manifestBlockIncrAgeMapDefault,
     .ageMapSize = LENGTH_OF(manifestBlockIncrAgeMapDefault),
+    .checksumSizeMap = manifestBlockIncrChecksumSizeMapDefault,
+    .checksumSizeMapSize = LENGTH_OF(manifestBlockIncrChecksumSizeMapDefault),
 };
 
 // Convert map size
@@ -337,6 +350,37 @@ backupBlockIncrMapSize(ConfigOption optionId, unsigned int optionKeyIdx, const S
     FUNCTION_TEST_RETURN(UINT, result);
 }
 
+// Convert map checksum size
+static unsigned int
+backupBlockIncrMapChecksumSize(ConfigOption optionId, unsigned int optionKeyIdx, const Variant *const value)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(ENUM, optionId);
+        FUNCTION_TEST_PARAM(UINT, optionKeyIdx);
+        FUNCTION_TEST_PARAM(VARIANT, value);
+    FUNCTION_TEST_END();
+
+    unsigned int result = 0;
+
+    TRY_BEGIN()
+    {
+        result = varUIntForce(value);
+    }
+    CATCH_ANY()
+    {
+    }
+    TRY_END();
+
+    if (result < BLOCK_INCR_CHECKSUM_SIZE_MIN)
+    {
+        THROW_FMT(
+            OptionInvalidValueError, "'%s' is not valid for '%s' option", strZ(varStr(value)),
+            cfgParseOptionKeyIdxName(optionId, optionKeyIdx));
+    }
+
+    FUNCTION_TEST_RETURN(UINT, result);
+}
+
 static ManifestBlockIncrMap
 backupBlockIncrMap(void)
 {
@@ -404,6 +448,36 @@ backupBlockIncrMap(void)
             result.ageMap = lstGet(map, 0);
             result.ageMapSize = lstSize(map);
         }
+
+        // Build checksum size map
+        const KeyValue *const manifestBlockIncrChecksumSizeKv = cfgOptionKvNull(cfgOptRepoBlockChecksumSizeMap);
+
+        if (manifestBlockIncrChecksumSizeKv != NULL)
+        {
+            List *const map = lstNewP(sizeof(ManifestBlockIncrChecksumSizeMap), .comparator = lstComparatorUInt);
+            const VariantList *const mapKeyList = kvKeyList(manifestBlockIncrChecksumSizeKv);
+
+            for (unsigned int mapKeyIdx = 0; mapKeyIdx < varLstSize(mapKeyList); mapKeyIdx++)
+            {
+                const Variant *mapKey = varLstGet(mapKeyList, mapKeyIdx);
+
+                ManifestBlockIncrChecksumSizeMap manifestBuildBlockIncrChecksumSizeMap =
+                {
+                    .blockSize = backupBlockIncrMapSize(
+                        cfgOptRepoBlockSizeMap, cfgOptionIdxDefault(cfgOptRepoBlockChecksumSizeMap), varStr(mapKey)),
+                    .checksumSize = backupBlockIncrMapChecksumSize(
+                        cfgOptRepoBlockSizeMap, cfgOptionIdxDefault(cfgOptRepoBlockChecksumSizeMap),
+                        kvGet(manifestBlockIncrChecksumSizeKv, mapKey)),
+                };
+
+                lstAdd(map, &manifestBuildBlockIncrChecksumSizeMap);
+            }
+
+            lstSort(map, sortOrderDesc);
+
+            result.checksumSizeMap = lstGet(map, 0);
+            result.checksumSizeMapSize = lstSize(map);
+        }
     }
 
     FUNCTION_TEST_RETURN_TYPE(ManifestBlockIncrMap, result);
@@ -1914,6 +1988,7 @@ backupJobCallback(void *data, unsigned int clientIdx)
                 if (blockIncr)
                 {
                     pckWriteU64P(param, file.blockIncrSize);
+                    pckWriteU64P(param, file.blockIncrChecksumSize);
                     pckWriteU64P(param, jobData->blockIncrSizeSuper);
 
                     if (file.blockIncrMapSize != 0)

src/command/backup/blockIncr.c

@@ -7,7 +7,7 @@ Block Incremental Filter
 #include "command/backup/blockMap.h"
 #include "common/compress/helper.h"
 #include "common/crypto/cipherBlock.h"
-#include "common/crypto/hash.h"
+#include "common/crypto/xxhash.h"
 #include "common/debug.h"
 #include "common/io/bufferRead.h"
 #include "common/io/bufferWrite.h"
@@ -39,6 +39,7 @@ typedef struct BlockIncr
     uint64_t blockOffset;                                           // Block offset
     uint64_t superBlockSize;                                        // Super block
     size_t blockSize;                                               // Block size
+    size_t checksumSize;                                            // Checksum size
     Buffer *block;                                                  // Block buffer
 
     Buffer *blockOut;                                               // Block output buffer
@@ -124,7 +125,7 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output)
             MEM_CONTEXT_TEMP_BEGIN()
             {
                 // Get block checksum
-                const Buffer *const checksum = cryptoHashOne(hashTypeSha1, this->block);
+                const Buffer *const checksum = xxHashOne(this->checksumSize, this->block);
 
                 // Does the block exist in the input map?
                 const BlockMapItem *const blockMapItemIn =
@@ -132,8 +133,7 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output)
                         blockMapGet(this->blockMapPrior, this->blockNo) : NULL;
 
                 // If the block is new or has changed then write it
-                if (blockMapItemIn == NULL ||
-                    memcmp(blockMapItemIn->checksum, bufPtrConst(checksum), bufUsed(checksum)) != 0)
+                if (blockMapItemIn == NULL || memcmp(blockMapItemIn->checksum, bufPtrConst(checksum), this->checksumSize) != 0)
                 {
                     // Begin the super block
                     if (this->blockOutWrite == NULL)
@@ -273,7 +273,7 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output)
 
                 // Write the map
                 ioWriteOpen(write);
-                blockMapWrite(this->blockMapOut, write, this->blockSize == this->superBlockSize);
+                blockMapWrite(this->blockMapOut, write, this->blockSize == this->superBlockSize, this->checksumSize);
                 ioWriteClose(write);
 
                 // Get total bytes written for the map
@@ -382,12 +382,14 @@ blockIncrInputSame(const THIS_VOID)
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
 blockIncrNew(
-    const uint64_t superBlockSize, const size_t blockSize, const unsigned int reference, const uint64_t bundleId,
-    const uint64_t bundleOffset, const Buffer *const blockMapPrior, const IoFilter *const compress, const IoFilter *const encrypt)
+    const uint64_t superBlockSize, const size_t blockSize, const size_t checksumSize, const unsigned int reference,
+    const uint64_t bundleId, const uint64_t bundleOffset, const Buffer *const blockMapPrior, const IoFilter *const compress,
+    const IoFilter *const encrypt)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(UINT64, superBlockSize);
         FUNCTION_LOG_PARAM(SIZE, blockSize);
+        FUNCTION_LOG_PARAM(SIZE, checksumSize);
         FUNCTION_LOG_PARAM(UINT, reference);
         FUNCTION_LOG_PARAM(UINT64, bundleId);
         FUNCTION_LOG_PARAM(UINT64, bundleOffset);
@@ -407,6 +409,7 @@ blockIncrNew(
             .memContext = memContextCurrent(),
             .superBlockSize = superBlockSize,
             .blockSize = blockSize,
+            .checksumSize = checksumSize,
             .reference = reference,
             .bundleId = bundleId,
             .blockOffset = bundleOffset,
@@ -439,7 +442,7 @@ blockIncrNew(
 
                 MEM_CONTEXT_PRIOR_BEGIN()
                 {
-                    driver->blockMapPrior = blockMapNewRead(read);
+                    driver->blockMapPrior = blockMapNewRead(read, checksumSize);
                 }
                 MEM_CONTEXT_PRIOR_END();
             }
@@ -455,6 +458,7 @@ blockIncrNew(
 
             pckWriteU64P(packWrite, driver->superBlockSize);
             pckWriteU64P(packWrite, blockSize);
+            pckWriteU64P(packWrite, checksumSize);
             pckWriteU32P(packWrite, reference);
             pckWriteU64P(packWrite, bundleId);
             pckWriteU64P(packWrite, bundleOffset);
@@ -491,6 +495,7 @@ blockIncrNewPack(const Pack *const paramList)
         PackRead *const paramListPack = pckReadNew(paramList);
         const uint64_t superBlockSize = pckReadU64P(paramListPack);
         const size_t blockSize = (size_t)pckReadU64P(paramListPack);
+        const size_t checksumSize = (size_t)pckReadU64P(paramListPack);
         const unsigned int reference = pckReadU32P(paramListPack);
         const uint64_t bundleId = pckReadU64P(paramListPack);
         const uint64_t bundleOffset = pckReadU64P(paramListPack);
@@ -511,7 +516,8 @@ blockIncrNewPack(const Pack *const paramList)
             encrypt = cipherBlockNewPack(encryptParam);
 
         result = ioFilterMove(
-            blockIncrNew(superBlockSize, blockSize, reference, bundleId, bundleOffset, blockMapPrior, compress, encrypt),
+            blockIncrNew(
+                superBlockSize, blockSize, checksumSize, reference, bundleId, bundleOffset, blockMapPrior, compress, encrypt),
             memContextPrior());
     }
     MEM_CONTEXT_TEMP_END();

src/command/backup/blockIncr.h

@@ -29,6 +29,14 @@ The super block list is followed by the block map, which is encrypted separately
 of the filter is the stored block map size. Combined with the repo size this allows the block map to be read separately.
 
 The block incremental should be read using BlockDelta since reconstructing the delta is quite involved.
+
+The xxHash algorithm is used to determine which blocks have changed. A 128-bit xxHash is generated and then checksumSize bytes are
+used from the hash depending on the size of the block. xxHash claims to have excellent dispersion characteristics, which has been
+verified by testing with SMHasher and a custom test suite. xxHash-32 is used for up to 4MiB content blocks in lz4 and the lower
+32-bits of xxHash-64 is used for content blocks in zstd. In general 32-bit checksums are pretty common defaults across filesystems
+such as BtrFS and ZFS. We use at least 5 bytes even for the smallest blocks since we are looking for changes and not just
+corruption. Ultimately if there is a collision and a block change is not detected it will almost certainly be caught by the overall
+SHA1 file checksum. This will fail the backup, which is not ideal, but better than restoring corrupted data.
 ***********************************************************************************************************************************/
 #ifndef COMMAND_BACKUP_BLOCK_INCR_H
 #define COMMAND_BACKUP_BLOCK_INCR_H
@@ -50,8 +58,8 @@ Constants needed to read the block number in a super block
 Constructors
 ***********************************************************************************************************************************/
 FN_EXTERN IoFilter *blockIncrNew(
-    uint64_t superBlockSize, size_t blockSize, unsigned int reference, uint64_t bundleId, uint64_t bundleOffset,
-    const Buffer *blockMapPrior, const IoFilter *compress, const IoFilter *encrypt);
+    uint64_t superBlockSize, size_t blockSize, size_t checksumSize, unsigned int reference, uint64_t bundleId,
+    uint64_t bundleOffset, const Buffer *blockMapPrior, const IoFilter *compress, const IoFilter *encrypt);
 FN_EXTERN IoFilter *blockIncrNewPack(const Pack *paramList);
 
 #endif

src/command/backup/blockMap.c

@@ -80,7 +80,7 @@ lstComparatorBlockMapReference(const void *const blockMapRef1, const void *const
 }
 
 FN_EXTERN BlockMap *
-blockMapNewRead(IoRead *const map)
+blockMapNewRead(IoRead *const map, size_t checksumSize)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(IO_READ, map);
@@ -97,7 +97,7 @@ blockMapNewRead(IoRead *const map)
     // Read all references in packed format
     BlockMap *const this = blockMapNew();
     List *const refList = lstNewP(sizeof(BlockMapReference), .comparator = lstComparatorBlockMapReference);
-    Buffer *const checksum = bufNew(HASH_TYPE_SHA1_SIZE);
+    Buffer *const checksum = bufNew(checksumSize);
     int64_t sizeLast = 0;
     bool referenceContinue = false;
 
@@ -247,12 +247,13 @@ blockMapNewRead(IoRead *const map)
 
 /**********************************************************************************************************************************/
 FN_EXTERN void
-blockMapWrite(const BlockMap *const this, IoWrite *const output, bool blockEqual)
+blockMapWrite(const BlockMap *const this, IoWrite *const output, const bool blockEqual, const size_t checksumSize)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(BLOCK_MAP, this);
         FUNCTION_LOG_PARAM(IO_WRITE, output);
         FUNCTION_LOG_PARAM(BOOL, blockEqual);
+        FUNCTION_LOG_PARAM(SIZE, checksumSize);
     FUNCTION_LOG_END();
 
     ASSERT(this != NULL);
@@ -422,7 +423,7 @@ blockMapWrite(const BlockMap *const this, IoWrite *const output, bool blockEqual
                 }
 
                 // Write checksum
-                ioWrite(output, BUF(block->checksum, HASH_TYPE_SHA1_SIZE));
+                ioWrite(output, BUF(block->checksum, checksumSize));
 
                 blockIdx++;
             }

src/command/backup/blockMap.h

@@ -13,18 +13,18 @@ Object type
 ***********************************************************************************************************************************/
 typedef struct BlockMap BlockMap;
 
-#include "common/crypto/hash.h"
+#include "common/crypto/xxhash.h"
 #include "common/type/list.h"
 #include "common/type/object.h"
 
 typedef struct BlockMapItem
 {
     unsigned int reference;                                         // Reference to backup where the block is stored
-    unsigned char checksum[HASH_TYPE_SHA1_SIZE];                    // Checksum of the block
     uint64_t bundleId;                                              // Bundle where the block is stored (0 if not bundled)
     uint64_t offset;                                                // Offset of super block into the bundle
     uint64_t size;                                                  // Size of the super block (including compression, etc.)
     uint64_t block;                                                 // Block no inside of super block
+    unsigned char checksum[XX_HASH_SIZE_MAX];                       // Checksum of the block
 } BlockMapItem;
 
 /***********************************************************************************************************************************
@@ -38,7 +38,7 @@ blockMapNew(void)
 }
 
 // New block map from IO
-FN_EXTERN BlockMap *blockMapNewRead(IoRead *map);
+FN_EXTERN BlockMap *blockMapNewRead(IoRead *map, size_t checksumSize);
 
 /***********************************************************************************************************************************
 Functions
@@ -52,7 +52,7 @@ blockMapAdd(BlockMap *const this, const BlockMapItem *const item)
 }
 
 // Write map to IO
-FN_EXTERN void blockMapWrite(const BlockMap *this, IoWrite *output, bool blockEqual);
+FN_EXTERN void blockMapWrite(const BlockMap *this, IoWrite *output, bool blockEqual, size_t checksumSize);
 
 /***********************************************************************************************************************************
 Getters/Setters

src/command/backup/file.c

@@ -254,8 +254,8 @@ backupFile(
                         ioFilterGroupAdd(
                             ioReadFilterGroup(storageReadIo(read)),
                             blockIncrNew(
-                                file->blockIncrSuperSize, file->blockIncrSize, blockIncrReference, bundleId, bundleOffset, blockMap,
-                                compress, encrypt));
+                                file->blockIncrSuperSize, file->blockIncrSize, file->blockIncrChecksumSize, blockIncrReference,
+                                bundleId, bundleOffset, blockMap, compress, encrypt));
 
                         repoChecksum = true;
                     }

src/command/backup/file.h

@@ -34,6 +34,7 @@ typedef struct BackupFile
     const Buffer *pgFileChecksum;                                   // Expected pg file checksum
     bool pgFileChecksumPage;                                        // Validate page checksums?
     size_t blockIncrSize;                                           // Perform block incremental on this file?
+    size_t blockIncrChecksumSize;                                   // Block checksum size
     uint64_t blockIncrSuperSize;                                    // Size of the super block
     const String *blockIncrMapPriorFile;                            // File containing prior block incremental map (NULL if none)
     uint64_t blockIncrMapPriorOffset;                               // Offset of prior block incremental map

src/command/backup/protocol.c

@@ -53,6 +53,7 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
 
             if (file.blockIncrSize > 0)
             {
+                file.blockIncrChecksumSize = (size_t)pckReadU64P(param);
                 file.blockIncrSuperSize = pckReadU64P(param);
                 file.blockIncrMapPriorFile = pckReadStrP(param);
 

src/command/restore/blockDelta.c

@@ -23,13 +23,14 @@ typedef struct BlockDeltaBlock
 {
     uint64_t no;                                                    // Block number in the super block
     uint64_t offset;                                                // Offset into original file
-    unsigned char checksum[HASH_TYPE_SHA1_SIZE];                    // Checksum of the block
+    unsigned char checksum[XX_HASH_SIZE_MAX];                       // Checksum of the block
 } BlockDeltaBlock;
 
 struct BlockDelta
 {
     BlockDeltaPub pub;                                              // Publicly accessible variables
     size_t blockSize;                                               // Block size
+    size_t checksumSize;                                            // Checksum size
     CipherType cipherType;                                          // Cipher type
     String *cipherPass;                                             // Cipher passphrase
     CompressType compressType;                                      // Compress type
@@ -53,12 +54,13 @@ typedef struct BlockDeltaReference
 
 FN_EXTERN BlockDelta *
 blockDeltaNew(
-    const BlockMap *const blockMap, const size_t blockSize, const Buffer *const blockHash, const CipherType cipherType,
-    const String *const cipherPass, const CompressType compressType)
+    const BlockMap *const blockMap, const size_t blockSize, const size_t checksumSize, const Buffer *const blockHash,
+    const CipherType cipherType, const String *const cipherPass, const CompressType compressType)
 {
     FUNCTION_TEST_BEGIN();
         FUNCTION_TEST_PARAM(BLOCK_MAP, blockMap);
         FUNCTION_TEST_PARAM(SIZE, blockSize);
+        FUNCTION_TEST_PARAM(SIZE, checksumSize);
         FUNCTION_TEST_PARAM(BUFFER, blockHash);
         FUNCTION_TEST_PARAM(STRING_ID, cipherType);
         FUNCTION_TEST_PARAM(STRING, cipherPass);
@@ -83,6 +85,7 @@ blockDeltaNew(
                 .readList = lstNewP(sizeof(BlockDeltaRead)),
             },
             .blockSize = blockSize,
+            .checksumSize = checksumSize,
             .cipherType = cipherType,
             .cipherPass = strDup(cipherPass),
             .compressType = compressType,
@@ -96,7 +99,7 @@ blockDeltaNew(
         {
             // Build list of references and for each reference the list of blocks for that reference
             const unsigned int blockHashSize =
-                blockHash == NULL ? 0 : (unsigned int)(bufUsed(blockHash) / HASH_TYPE_SHA1_SIZE);
+                blockHash == NULL ? 0 : (unsigned int)(bufUsed(blockHash) / this->checksumSize);
             List *const referenceList = lstNewP(sizeof(BlockDeltaReference), .comparator = lstComparatorUInt);
 
             for (unsigned int blockMapIdx = 0; blockMapIdx < blockMapSize(blockMap); blockMapIdx++)
@@ -107,8 +110,8 @@ blockDeltaNew(
                 // stored in the repository is different from the block hash list
                 if (blockMapIdx >= blockHashSize ||
                     !bufEq(
-                        BUF(blockMapItem->checksum, HASH_TYPE_SHA1_SIZE),
-                        BUF(bufPtrConst(blockHash) + blockMapIdx * HASH_TYPE_SHA1_SIZE, HASH_TYPE_SHA1_SIZE)))
+                        BUF(blockMapItem->checksum, this->checksumSize),
+                        BUF(bufPtrConst(blockHash) + blockMapIdx * this->checksumSize, this->checksumSize)))
                 {
                     const unsigned int reference = blockMapItem->reference;
                     BlockDeltaReference *const referenceData = lstFind(referenceList, &reference);

src/command/restore/blockDelta.h

@@ -9,6 +9,7 @@ because the file to restore may not exist so all the blocks will need to be rest
 
 #include "command/backup/blockMap.h"
 #include "common/compress/helper.h"
+#include "common/crypto/common.h"
 
 /***********************************************************************************************************************************
 Object type
@@ -36,8 +37,8 @@ typedef struct BlockDeltaWrite
 Constructors
 ***********************************************************************************************************************************/
 FN_EXTERN BlockDelta *blockDeltaNew(
-    const BlockMap *blockMap, size_t blockSize, const Buffer *blockHash, CipherType cipherType, const String *cipherPass,
-    const CompressType compressType);
+    const BlockMap *blockMap, size_t blockSize, size_t checksumSize, const Buffer *blockHash, CipherType cipherType,
+    const String *cipherPass, const CompressType compressType);
 
 /***********************************************************************************************************************************
 Functions

src/command/restore/blockHash.c

@@ -5,7 +5,7 @@ Restore Delta Map
 
 #include "command/restore/blockHash.h"
 #include "common/crypto/common.h"
-#include "common/crypto/hash.h"
+#include "common/crypto/xxhash.h"
 #include "common/debug.h"
 #include "common/log.h"
 #include "common/type/object.h"
@@ -18,6 +18,7 @@ typedef struct BlockHash
     MemContext *memContext;                                         // Mem context of filter
 
     size_t blockSize;                                               // Block size for checksums
+    size_t checksumSize;                                            // Checksum size
     size_t blockCurrent;                                            // Size of current block
     IoFilter *hash;                                                 // Hash of current block
     List *list;                                                     // List of hashes
@@ -57,7 +58,7 @@ blockHashProcess(THIS_VOID, const Buffer *const input)
         {
             MEM_CONTEXT_BEGIN(this->memContext)
             {
-                this->hash = cryptoHashNew(hashTypeSha1);
+                this->hash = xxHashNew(this->checksumSize);
                 this->blockCurrent = 0;
             }
             MEM_CONTEXT_END();
@@ -115,7 +116,7 @@ blockHashResult(THIS_VOID)
         if (this->hash)
             lstAdd(this->list, bufPtrConst(pckReadBinP(pckReadNew(ioFilterResult(this->hash)))));
 
-        pckWriteBinP(packWrite, BUF(lstGet(this->list, 0), lstSize(this->list) * HASH_TYPE_SHA1_SIZE));
+        pckWriteBinP(packWrite, BUF(lstGet(this->list, 0), lstSize(this->list) * this->checksumSize));
         pckWriteEndP(packWrite);
 
         result = pckMove(pckWriteResult(packWrite), memContextPrior());
@@ -127,10 +128,11 @@ blockHashResult(THIS_VOID)
 
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
-blockHashNew(const size_t blockSize)
+blockHashNew(const size_t blockSize, const size_t checksumSize)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(SIZE, blockSize);
+        FUNCTION_LOG_PARAM(SIZE, checksumSize);
     FUNCTION_LOG_END();
 
     ASSERT(blockSize != 0);
@@ -146,7 +148,8 @@ blockHashNew(const size_t blockSize)
         {
             .memContext = memContextCurrent(),
             .blockSize = blockSize,
-            .list = lstNewP(HASH_TYPE_SHA1_SIZE),
+            .checksumSize = checksumSize,
+            .list = lstNewP(checksumSize),
         };
 
         this = ioFilterNewP(BLOCK_HASH_FILTER_TYPE, driver, NULL, .in = blockHashProcess, .result = blockHashResult);

src/command/restore/blockHash.h

@@ -18,6 +18,6 @@ Filter type constant
 /***********************************************************************************************************************************
 Constructors
 ***********************************************************************************************************************************/
-FN_EXTERN IoFilter *blockHashNew(size_t blockSize);
+FN_EXTERN IoFilter *blockHashNew(size_t blockSize, size_t checksumSize);
 
 #endif

src/command/restore/file.c

@@ -127,7 +127,11 @@ restoreFile(
 
                                     // Generate block hash list if block incremental
                                     if (file->blockIncrMapSize != 0)
-                                        ioFilterGroupAdd(ioReadFilterGroup(read), blockHashNew(file->blockIncrSize));
+                                    {
+                                        ioFilterGroupAdd(
+                                            ioReadFilterGroup(read),
+                                            blockHashNew(file->blockIncrSize, file->blockIncrChecksumSize));
+                                    }
 
                                     ioReadDrain(read);
                                 }
@@ -291,7 +295,7 @@ restoreFile(
 
                         // Read block map. This will be compared to the block hash list already created to determine which blocks
                         // need to be fetched from the repository. If we got here there must be at least one block to fetch.
-                        const BlockMap *const blockMap = blockMapNewRead(storageReadIo(repoFileRead));
+                        const BlockMap *const blockMap = blockMapNewRead(storageReadIo(repoFileRead), file->blockIncrChecksumSize);
 
                         // The repo file needs to be closed so that block lists can be read from the remote protocol
                         ioReadClose(storageReadIo(repoFileRead));
@@ -301,7 +305,7 @@ restoreFile(
 
                         // Apply delta to file
                         BlockDelta *const blockDelta = blockDeltaNew(
-                            blockMap, file->blockIncrSize, file->blockHash,
+                            blockMap, file->blockIncrSize, file->blockIncrChecksumSize, file->blockHash,
                             cipherPass == NULL ? cipherTypeNone : cipherTypeAes256Cbc, cipherPass, repoFileCompressType);
 
                         for (unsigned int readIdx = 0; readIdx < blockDeltaReadSize(blockDelta); readIdx++)

src/command/restore/file.h

@@ -35,6 +35,7 @@ typedef struct RestoreFile
     const Variant *limit;                                           // Limit for read in the repo file
     uint64_t blockIncrMapSize;                                      // Block incremental map size (0 if not incremental)
     size_t blockIncrSize;                                           // Block incremental size (when map size > 0)
+    size_t blockIncrChecksumSize;                                   // Checksum size (when map size > 0)
     const String *manifestFile;                                     // Manifest file
     const Buffer *blockHash;                                        // Hashes for block incremental restore, set in restoreFile()
 } RestoreFile;

src/command/restore/protocol.c

@@ -61,7 +61,10 @@ restoreFileProtocol(PackRead *const param, ProtocolServer *const server)
             file.blockIncrMapSize = pckReadU64P(param);
 
             if (file.blockIncrMapSize != 0)
+            {
                 file.blockIncrSize = (size_t)pckReadU64P(param);
+                file.blockIncrChecksumSize = (size_t)pckReadU64P(param);
+            }
 
             file.manifestFile = pckReadStrP(param);
 

src/command/restore/restore.c

@@ -2386,7 +2386,10 @@ restoreJobCallback(void *data, unsigned int clientIdx)
                 pckWriteU64P(param, file.blockIncrMapSize);
 
                 if (file.blockIncrMapSize != 0)
+                {
                     pckWriteU64P(param, file.blockIncrSize);
+                    pckWriteU64P(param, file.blockIncrChecksumSize);
+                }
 
                 pckWriteStrP(param, file.name);
 

src/common/crypto/xxhash.c

@@ -0,0 +1,162 @@
+/***********************************************************************************************************************************
+xxHash Interface
+***********************************************************************************************************************************/
+#include "build.auto.h"
+
+#include "common/crypto/xxhash.h"
+#include "common/debug.h"
+#include "common/io/filter/filter.h"
+#include "common/log.h"
+#include "common/type/object.h"
+
+/***********************************************************************************************************************************
+Include local xxHash code
+***********************************************************************************************************************************/
+#define XXH_INLINE_ALL
+
+#include "common/crypto/xxhash.vendor.c.inc"
+
+/***********************************************************************************************************************************
+Object type
+***********************************************************************************************************************************/
+typedef struct XxHash
+{
+    size_t size;                                                    // Size of hash to return
+    XXH3_state_t *state;                                            // xxHash state
+} XxHash;
+
+/***********************************************************************************************************************************
+Macros for function logging
+***********************************************************************************************************************************/
+#define FUNCTION_LOG_XX_HASH_TYPE                                                                                              \
+    XxHash *
+#define FUNCTION_LOG_XX_HASH_FORMAT(value, buffer, bufferSize)                                                                 \
+    objNameToLog(value, "XxHash", buffer, bufferSize)
+
+/***********************************************************************************************************************************
+Free hash context
+***********************************************************************************************************************************/
+static void
+xxHashFreeResource(THIS_VOID)
+{
+    THIS(XxHash);
+
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(XX_HASH, this);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+
+    XXH3_freeState(this->state);
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/***********************************************************************************************************************************
+Add message data to the hash from a Buffer
+***********************************************************************************************************************************/
+static void
+xxHashProcess(THIS_VOID, const Buffer *const message)
+{
+    THIS(XxHash);
+
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(XX_HASH, this);
+        FUNCTION_LOG_PARAM(BUFFER, message);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+    ASSERT(message != NULL);
+
+    XXH3_128bits_update(this->state, bufPtrConst(message), bufUsed(message));
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/***********************************************************************************************************************************
+Get string representation of the hash as a filter result
+***********************************************************************************************************************************/
+static Pack *
+xxHashResult(THIS_VOID)
+{
+    THIS(XxHash);
+
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(XX_HASH, this);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+
+    Pack *result = NULL;
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        PackWrite *const packWrite = pckWriteNewP();
+
+        XXH128_canonical_t canonical;
+        XXH128_canonicalFromHash(&canonical, XXH3_128bits_digest(this->state));
+
+        pckWriteBinP(packWrite, BUF(canonical.digest, this->size));
+        pckWriteEndP(packWrite);
+
+        result = pckMove(pckWriteResult(packWrite), memContextPrior());
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN(PACK, result);
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN IoFilter *
+xxHashNew(const size_t size)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(SIZE, size);
+    FUNCTION_LOG_END();
+
+    ASSERT(size >= 1 && size <= XX_HASH_SIZE_MAX);
+
+    // Allocate memory to hold process state
+    IoFilter *this = NULL;
+
+    OBJ_NEW_BEGIN(XxHash, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
+    {
+        XxHash *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), IoFilter::XxHash);
+        *driver = (XxHash){.size = size};
+
+        driver->state = XXH3_createState();
+        XXH3_128bits_reset(driver->state);
+
+        // Set free callback to ensure hash context is freed
+        memContextCallbackSet(objMemContext(driver), xxHashFreeResource, driver);
+
+        // Create filter interface
+        this = ioFilterNewP(XX_HASH_FILTER_TYPE, driver, NULL, .in = xxHashProcess, .result = xxHashResult);
+    }
+    OBJ_NEW_END();
+
+    FUNCTION_LOG_RETURN(IO_FILTER, this);
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN Buffer *
+xxHashOne(const size_t size, const Buffer *const message)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(SIZE, size);
+        FUNCTION_LOG_PARAM(BUFFER, message);
+    FUNCTION_LOG_END();
+
+    ASSERT(size >= 1 && size <= XX_HASH_SIZE_MAX);
+    ASSERT(message != NULL);
+
+    Buffer *const result = bufNew(size);
+
+    XXH128_canonical_t canonical;
+    XXH128_canonicalFromHash(&canonical, XXH3_128bits(bufPtrConst(message), bufUsed(message)));
+
+    memcpy(bufPtr(result), canonical.digest, size);
+    bufUsedSet(result, size);
+
+    FUNCTION_LOG_RETURN(BUFFER, result);
+}

src/common/crypto/xxhash.h

@@ -0,0 +1,38 @@
+/***********************************************************************************************************************************
+xxHash Interface
+
+Generate a 128-bit hash and return the specified number of bytes from the canonical representation. The 128-bit variant is only
+slightly slower than the 64-bit variant so this makes getting up to 16 bytes of hash value fairly simple.
+
+We consider this safe because xxHash claims to have excellent dispersion, zstd uses the lower 32-bits of a 64-bit hash for content
+checksums, xxHash has been tested with SMHasher, and the "128-bit variant has better mixing and strength than the 64-bit variant,
+even without counting the significantly larger output size".
+***********************************************************************************************************************************/
+#ifndef COMMON_CRYPTO_XXHASH_H
+#define COMMON_CRYPTO_XXHASH_H
+
+#include "common/io/filter/filter.h"
+#include "common/type/buffer.h"
+
+/***********************************************************************************************************************************
+Filter type constant
+***********************************************************************************************************************************/
+#define XX_HASH_FILTER_TYPE                                         STRID5("xxhash", 0x1130a3180)
+
+/***********************************************************************************************************************************
+Max size of hash
+***********************************************************************************************************************************/
+#define XX_HASH_SIZE_MAX                                            16
+
+/***********************************************************************************************************************************
+Constructors
+***********************************************************************************************************************************/
+FN_EXTERN IoFilter *xxHashNew(size_t size);
+
+/***********************************************************************************************************************************
+Helper functions
+***********************************************************************************************************************************/
+// Get hash for one buffer
+FN_EXTERN Buffer *xxHashOne(size_t size, const Buffer *message);
+
+#endif

src/config/config.auto.h

@@ -131,7 +131,7 @@ Option constants
 #define CFGOPT_TYPE                                                 "type"
 #define CFGOPT_VERBOSE                                              "verbose"
 
-#define CFG_OPTION_TOTAL                                            163
+#define CFG_OPTION_TOTAL                                            164
 
 /***********************************************************************************************************************************
 Option value constants
@@ -447,6 +447,7 @@ typedef enum
     cfgOptRepoAzureUriStyle,
     cfgOptRepoBlock,
     cfgOptRepoBlockAgeMap,
+    cfgOptRepoBlockChecksumSizeMap,
     cfgOptRepoBlockSizeMap,
     cfgOptRepoBlockSizeSuper,
     cfgOptRepoBlockSizeSuperFull,

src/config/parse.auto.c.inc

@@ -4969,6 +4969,35 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
         ),                                                                                                 // opt/repo-block-age-map
     ),                                                                                                     // opt/repo-block-age-map
     // -----------------------------------------------------------------------------------------------------------------------------
+    PARSE_RULE_OPTION                                                                            // opt/repo-block-checksum-size-map
+    (                                                                                            // opt/repo-block-checksum-size-map
+        PARSE_RULE_OPTION_NAME("repo-block-checksum-size-map"),                                  // opt/repo-block-checksum-size-map
+        PARSE_RULE_OPTION_TYPE(cfgOptTypeHash),                                                  // opt/repo-block-checksum-size-map
+        PARSE_RULE_OPTION_RESET(true),                                                           // opt/repo-block-checksum-size-map
+        PARSE_RULE_OPTION_REQUIRED(false),                                                       // opt/repo-block-checksum-size-map
+        PARSE_RULE_OPTION_SECTION(cfgSectionGlobal),                                             // opt/repo-block-checksum-size-map
+        PARSE_RULE_OPTION_MULTI(true),                                                           // opt/repo-block-checksum-size-map
+        PARSE_RULE_OPTION_GROUP_MEMBER(true),                                                    // opt/repo-block-checksum-size-map
+        PARSE_RULE_OPTION_GROUP_ID(cfgOptGrpRepo),                                               // opt/repo-block-checksum-size-map
+                                                                                                 // opt/repo-block-checksum-size-map
+        PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST                                           // opt/repo-block-checksum-size-map
+        (                                                                                        // opt/repo-block-checksum-size-map
+            PARSE_RULE_OPTION_COMMAND(cfgCmdBackup)                                              // opt/repo-block-checksum-size-map
+        ),                                                                                       // opt/repo-block-checksum-size-map
+                                                                                                 // opt/repo-block-checksum-size-map
+        PARSE_RULE_OPTIONAL                                                                      // opt/repo-block-checksum-size-map
+        (                                                                                        // opt/repo-block-checksum-size-map
+            PARSE_RULE_OPTIONAL_GROUP                                                            // opt/repo-block-checksum-size-map
+            (                                                                                    // opt/repo-block-checksum-size-map
+                PARSE_RULE_OPTIONAL_DEPEND                                                       // opt/repo-block-checksum-size-map
+                (                                                                                // opt/repo-block-checksum-size-map
+                    PARSE_RULE_VAL_OPT(cfgOptRepoBlock),                                         // opt/repo-block-checksum-size-map
+                    PARSE_RULE_VAL_BOOL_TRUE,                                                    // opt/repo-block-checksum-size-map
+                ),                                                                               // opt/repo-block-checksum-size-map
+            ),                                                                                   // opt/repo-block-checksum-size-map
+        ),                                                                                       // opt/repo-block-checksum-size-map
+    ),                                                                                           // opt/repo-block-checksum-size-map
+    // -----------------------------------------------------------------------------------------------------------------------------
     PARSE_RULE_OPTION                                                                                     // opt/repo-block-size-map
     (                                                                                                     // opt/repo-block-size-map
         PARSE_RULE_OPTION_NAME("repo-block-size-map"),                                                    // opt/repo-block-size-map
@@ -9655,6 +9684,7 @@ static const uint8_t optionResolveOrder[] =
     cfgOptRepoAzureUriStyle,                                                                                    // opt-resolve-order
     cfgOptRepoBlock,                                                                                            // opt-resolve-order
     cfgOptRepoBlockAgeMap,                                                                                      // opt-resolve-order
+    cfgOptRepoBlockChecksumSizeMap,                                                                             // opt-resolve-order
     cfgOptRepoBlockSizeMap,                                                                                     // opt-resolve-order
     cfgOptRepoBlockSizeSuper,                                                                                   // opt-resolve-order
     cfgOptRepoBlockSizeSuperFull,                                                                               // opt-resolve-order

src/info/manifest.c

@@ -240,6 +240,7 @@ manifestFilePack(const Manifest *const manifest, const ManifestFile *const file)
         ASSERT(file->blockIncrSize % BLOCK_INCR_SIZE_FACTOR == 0);
 
         cvtUInt64ToVarInt128(file->blockIncrSize / BLOCK_INCR_SIZE_FACTOR, buffer, &bufferPos, sizeof(buffer));
+        cvtUInt64ToVarInt128(file->blockIncrChecksumSize, buffer, &bufferPos, sizeof(buffer));
         cvtUInt64ToVarInt128(file->blockIncrMapSize, buffer, &bufferPos, sizeof(buffer));
     }
 
@@ -367,6 +368,7 @@ manifestFileUnpack(const Manifest *const manifest, const ManifestFilePack *const
     {
         result.blockIncrSize =
             (size_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX) * BLOCK_INCR_SIZE_FACTOR;
+        result.blockIncrChecksumSize = (size_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX);
         result.blockIncrMapSize = cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX);
     }
 
@@ -815,6 +817,33 @@ manifestBuildBlockIncrSize(const ManifestBuildData *const buildData, const Manif
     FUNCTION_TEST_RETURN(UINT64, result);
 }
 
+// Get checksum size for a block size. Since these checksums are used to determine if a block has changed (not for corruption) they
+// should be set larger than usual. For example, it is common to use 32-bit checksums to check for corruption in blocks up to 4MiB
+// but here we used more bits to ensure block changes are correctly detected, since valid changes may look a lot different than
+// corruption.
+static size_t
+manifestBuildBlockIncrChecksumSize(const ManifestBuildData *const buildData, const size_t blockSize)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM_P(VOID, buildData);
+        FUNCTION_TEST_PARAM(SIZE, blockSize);
+    FUNCTION_TEST_END();
+
+    size_t result = BLOCK_INCR_CHECKSUM_SIZE_MIN;
+
+    // Search checksum size map for larger value
+    for (unsigned int sizeIdx = 0; sizeIdx < buildData->blockIncrMap->checksumSizeMapSize; sizeIdx++)
+    {
+        if (blockSize >= buildData->blockIncrMap->checksumSizeMap[sizeIdx].blockSize)
+        {
+            result = buildData->blockIncrMap->checksumSizeMap[sizeIdx].checksumSize;
+            break;
+        }
+    }
+
+    FUNCTION_TEST_RETURN(SIZE, result);
+}
+
 // Process files/links/paths and add them to the manifest
 static void
 manifestBuildInfo(
@@ -1036,7 +1065,10 @@ manifestBuildInfo(
 
             // Get block incremental size
             if (info->size != 0 && buildData->manifest->pub.data.blockIncr)
+            {
                 file.blockIncrSize = manifestBuildBlockIncrSize(buildData, &file);
+                file.blockIncrChecksumSize = manifestBuildBlockIncrChecksumSize(buildData, file.blockIncrSize);
+            }
 
             // Determine if this file should be page checksummed
             if (dbPath && buildData->checksumPage)
@@ -1650,6 +1682,7 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type
                     if (file.blockIncrSize > 0)
                     {
                         file.blockIncrSize = filePrior.blockIncrSize;
+                        file.blockIncrChecksumSize = filePrior.blockIncrChecksumSize;
                         file.blockIncrMapSize = filePrior.blockIncrMapSize;
                     }
 
@@ -1804,6 +1837,7 @@ manifestBuildComplete(
 #define MANIFEST_KEY_BACKUP_TIMESTAMP_STOP                          "backup-timestamp-stop"
 #define MANIFEST_KEY_BACKUP_TYPE                                    "backup-type"
 #define MANIFEST_KEY_BLOCK_INCR                                     STRID5("bi", 0x1220)
+#define MANIFEST_KEY_BLOCK_INCR_CHECKSUM                            STRID5("bic", 0xd220)
 #define MANIFEST_KEY_BLOCK_INCR_MAP                                 STRID5("bim", 0x35220)
 #define MANIFEST_KEY_BUNDLE_ID                                      STRID5("bni", 0x25c20)
 #define MANIFEST_KEY_BUNDLE_OFFSET                                  STRID5("bno", 0x3dc20)
@@ -1944,6 +1978,11 @@ manifestLoadCallback(void *callbackData, const String *const section, const Stri
         {
             file.blockIncrSize = (size_t)jsonReadUInt64(json) * BLOCK_INCR_SIZE_FACTOR;
 
+            if (jsonReadKeyExpectStrId(json, MANIFEST_KEY_BLOCK_INCR_CHECKSUM))
+                file.blockIncrChecksumSize = (size_t)jsonReadUInt64(json);
+            else
+                file.blockIncrChecksumSize = BLOCK_INCR_CHECKSUM_SIZE_MIN;
+
             if (jsonReadKeyExpectStrId(json, MANIFEST_KEY_BLOCK_INCR_MAP))
                 file.blockIncrMapSize = jsonReadUInt64(json);
         }
@@ -2679,6 +2718,9 @@ manifestSaveCallback(void *const callbackData, const String *const sectionNext,
                 {
                     jsonWriteUInt64(jsonWriteKeyStrId(json, MANIFEST_KEY_BLOCK_INCR), file.blockIncrSize / BLOCK_INCR_SIZE_FACTOR);
 
+                    if (file.blockIncrChecksumSize != BLOCK_INCR_CHECKSUM_SIZE_MIN)
+                        jsonWriteUInt64(jsonWriteKeyStrId(json, MANIFEST_KEY_BLOCK_INCR_CHECKSUM), file.blockIncrChecksumSize);
+
                     if (file.blockIncrMapSize != 0)
                         jsonWriteUInt64(jsonWriteKeyStrId(json, MANIFEST_KEY_BLOCK_INCR_MAP), file.blockIncrMapSize);
                 }

src/info/manifest.h

@@ -27,6 +27,9 @@ STRING_DECLARE(MANIFEST_TARGET_PGDATA_STR);
 #define MANIFEST_TARGET_PGTBLSPC                                    "pg_tblspc"
 STRING_DECLARE(MANIFEST_TARGET_PGTBLSPC_STR);
 
+// Minimum size for the block incremental checksum
+#define BLOCK_INCR_CHECKSUM_SIZE_MIN                                6
+
 /***********************************************************************************************************************************
 Object type
 ***********************************************************************************************************************************/
@@ -104,12 +107,21 @@ typedef struct ManifestBlockIncrAgeMap
     uint32_t blockMultiplier;                                       // Block multiplier
 } ManifestBlockIncrAgeMap;
 
+// Map block size to checksum size
+typedef struct ManifestBlockIncrChecksumSizeMap
+{
+    uint32_t blockSize;
+    uint32_t checksumSize;
+} ManifestBlockIncrChecksumSizeMap;
+
 typedef struct ManifestBlockIncrMap
 {
     const ManifestBlockIncrSizeMap *sizeMap;                        // Block size map
     unsigned int sizeMapSize;                                       // Block size map size
     const ManifestBlockIncrAgeMap *ageMap;                          // File age map
     unsigned int ageMapSize;                                        // File age map size
+    const ManifestBlockIncrChecksumSizeMap *checksumSizeMap;        // Checksum size map
+    unsigned int checksumSizeMapSize;                               // Checksum size map size
 } ManifestBlockIncrMap;
 
 /***********************************************************************************************************************************
@@ -143,6 +155,7 @@ typedef struct ManifestFile
     uint64_t bundleId;                                              // Bundle id
     uint64_t bundleOffset;                                          // Bundle offset
     size_t blockIncrSize;                                           // Size of incremental blocks
+    size_t blockIncrChecksumSize;                                   // Size of incremental block checksum
     uint64_t blockIncrMapSize;                                      // Block incremental map size
     uint64_t size;                                                  // Original size
     uint64_t sizeRepo;                                              // Size in repo

src/meson.build

@@ -177,6 +177,7 @@ src_pgbackrest = [
 	'common/crypto/cipherBlock.c',
 	'common/crypto/common.c',
 	'common/crypto/hash.c',
+	'common/crypto/xxhash.c',
 	'common/exec.c',
 	'common/fork.c',
 	'common/ini.c',

test/define.yaml

@@ -331,7 +331,7 @@ unit:
 
       # ----------------------------------------------------------------------------------------------------------------------------
       - name: crypto
-        total: 3
+        total: 4
         feature: STORAGE
         harness: storage
 
@@ -340,6 +340,7 @@ unit:
           - common/crypto/common
           - common/crypto/hash
           - common/crypto/md5.vendor: included
+          - common/crypto/xxhash
 
       # ----------------------------------------------------------------------------------------------------------------------------
       - name: io-tls

test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm

@@ -2020,6 +2020,8 @@ sub restoreCompare
         delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bno"});
         $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bi");
         delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bi"});
+        $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bic");
+        delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bic"});
         $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bim");
         delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bim"});
 

test/src/common/harnessManifest.c

@@ -84,6 +84,7 @@ hrnManifestFileAdd(Manifest *const manifest, const HrnManifestFile hrnManifestFi
             .bundleId = hrnManifestFile.bundleId,
             .bundleOffset = hrnManifestFile.bundleOffset,
             .blockIncrSize = hrnManifestFile.blockIncrSize,
+            .blockIncrChecksumSize = hrnManifestFile.blockIncrChecksumSize,
             .blockIncrMapSize = hrnManifestFile.blockIncrMapSize,
             .size = hrnManifestFile.size,
             .sizeRepo = hrnManifestFile.sizeRepo,

test/src/common/harnessManifest.h

@@ -43,6 +43,7 @@ typedef struct HrnManifestFile
     uint64_t bundleId;
     uint64_t bundleOffset;
     size_t blockIncrSize;
+    size_t blockIncrChecksumSize;
     uint64_t blockIncrMapSize;
     uint64_t size;
     uint64_t sizeRepo;

test/src/module/command/backupTest.c

@@ -24,18 +24,19 @@ Test Backup Command
 Test block delta
 ***********************************************************************************************************************************/
 static String *
-testBlockDelta(const BlockMap *const blockMap, const size_t blockSize)
+testBlockDelta(const BlockMap *const blockMap, const size_t blockSize, const size_t checksumSize)
 {
     FUNCTION_HARNESS_BEGIN();
         FUNCTION_HARNESS_PARAM(BLOCK_MAP, blockMap);
         FUNCTION_HARNESS_PARAM(SIZE, blockSize);
+        FUNCTION_HARNESS_PARAM(SIZE, checksumSize);
     FUNCTION_HARNESS_END();
 
     ASSERT(blockMap != NULL);
     ASSERT(blockSize > 0);
 
     String *const result = strNew();
-    BlockDelta *const blockDelta = blockDeltaNew(blockMap, blockSize, NULL, cipherTypeNone, NULL, compressTypeNone);
+    BlockDelta *const blockDelta = blockDeltaNew(blockMap, blockSize, checksumSize, NULL, cipherTypeNone, NULL, compressTypeNone);
 
     for (unsigned int readIdx = 0; readIdx < blockDeltaReadSize(blockDelta); readIdx++)
     {
@@ -204,7 +205,7 @@ testBackupValidateList(
 
                         ioReadOpen(storageReadIo(read));
 
-                        const BlockMap *const blockMap = blockMapNewRead(storageReadIo(read));
+                        const BlockMap *const blockMap = blockMapNewRead(storageReadIo(read), file.blockIncrChecksumSize);
 
                         // Build map log
                         String *const mapLog = strNew();
@@ -236,7 +237,8 @@ testBackupValidateList(
                         bufUsedSet(fileBuffer, bufSize(fileBuffer));
 
                         BlockDelta *const blockDelta = blockDeltaNew(
-                            blockMap, file.blockIncrSize, NULL, cipherType, cipherPass, manifestData->backupOptionCompressType);
+                            blockMap, file.blockIncrSize, file.blockIncrChecksumSize, NULL, cipherType, cipherPass,
+                            manifestData->backupOptionCompressType);
 
                         for (unsigned int readIdx = 0; readIdx < blockDeltaReadSize(blockDelta); readIdx++)
                         {
@@ -1007,7 +1009,7 @@ testRun(void)
             .bundleId = 0,
             .offset = 0,
             .size = 3,
-            .checksum = {0xee, 0xee, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x01, 0xff, 0xff},
         };
 
         TEST_RESULT_UINT(blockMapAdd(blockMap, &blockMapItem)->reference, 128, "add");
@@ -1019,7 +1021,7 @@ testRun(void)
             .bundleId = 0,
             .offset = 3,
             .size = 5,
-            .checksum = {0xee, 0xee, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x02, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1030,7 +1032,7 @@ testRun(void)
             .bundleId = 1,
             .offset = 1,
             .size = 5,
-            .checksum = {0xee, 0xee, 0x03, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x03, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1041,7 +1043,7 @@ testRun(void)
             .bundleId = 0,
             .offset = 8,
             .size = 99,
-            .checksum = {0xee, 0xee, 0x04, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x04, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1052,7 +1054,7 @@ testRun(void)
             .bundleId = 1,
             .offset = 7,
             .size = 99,
-            .checksum = {0xee, 0xee, 0x05, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x05, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1064,7 +1066,7 @@ testRun(void)
             .bundleId = 0,
             .offset = 0,
             .size = 8,
-            .checksum = {0xee, 0xee, 0x88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x88, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1074,7 +1076,7 @@ testRun(void)
 
         Buffer *buffer = bufNew(256);
         IoWrite *write = ioBufferWriteNewOpen(buffer);
-        TEST_RESULT_VOID(blockMapWrite(blockMap, write, true), "save");
+        TEST_RESULT_VOID(blockMapWrite(blockMap, write, true, 5), "save");
         ioWriteClose(write);
 
         TEST_RESULT_STR_Z(
@@ -1083,28 +1085,28 @@ testRun(void)
 
             "8008"                                      // reference 128
             "06"                                        // super block size 3
-            "eeee01000000000000000000000000000000ffff"  // checksum
+            "eeee01ffff"                                // checksum
             "09"                                        // super block size 5
-            "eeee02000000000000000000000000000000ffff"  // checksum
+            "eeee02ffff"                                // checksum
 
             "06"                                        // reference 0
             "01"                                        // bundle 1
             "01"                                        // offset 1
             "01"                                        // super block size 5
-            "eeee03000000000000000000000000000000ffff"  // checksum
+            "eeee03ffff"                                // checksum
 
             "8008"                                      // reference 128
             "f902"                                      // super block size 99
-            "eeee04000000000000000000000000000000ffff"  // checksum
+            "eeee04ffff"                                // checksum
 
             "04"                                        // reference 0
             "01"                                        // offset 7
             "01"                                        // super block size 99
-            "eeee05000000000000000000000000000000ffff"  // checksum
+            "eeee05ffff"                                // checksum
 
             "21"                                        // reference 0
             "eb02"                                      // super block size 8
-            "eeee88000000000000000000000000000000ffff", // reference
+            "eeee88ffff",                               // checksum
             "compare");
 
         // -------------------------------------------------------------------------------------------------------------------------
@@ -1112,7 +1114,7 @@ testRun(void)
 
         Buffer *bufferCompare = bufNew(256);
         write = ioBufferWriteNewOpen(bufferCompare);
-        TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer)), write, true), "read and save");
+        TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer), 5), write, true, 5), "read and save");
         ioWriteClose(write);
 
         TEST_RESULT_STR(strNewEncode(encodingHex, bufferCompare), strNewEncode(encodingHex, buffer), "compare");
@@ -1121,7 +1123,7 @@ testRun(void)
         TEST_TITLE("equal block delta");
 
         TEST_RESULT_STR_Z(
-            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer)), 8),
+            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer), 5), 8, 5),
             "read {reference: 128, bundleId: 0, offset: 0, size: 107}\n"
             "  super block {size: 3}\n"
             "    block {no: 0, offset: 0}\n"
@@ -1151,7 +1153,7 @@ testRun(void)
             .offset = 0,
             .size = 4,
             .block = 0,
-            .checksum = {0xee, 0xee, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x01, 0, 0, 0, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1162,7 +1164,7 @@ testRun(void)
             .offset = 0,
             .size = 4,
             .block = 1,
-            .checksum = {0xee, 0xee, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x02, 0, 0, 0, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1173,7 +1175,7 @@ testRun(void)
             .offset = 4,
             .size = 5,
             .block = 0,
-            .checksum = {0xee, 0xee, 0x03, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x03, 0, 0, 0, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1184,7 +1186,7 @@ testRun(void)
             .offset = 0,
             .size = 99,
             .block = 0,
-            .checksum = {0xee, 0xee, 0x04, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x04, 0, 0, 0, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1195,7 +1197,7 @@ testRun(void)
             .offset = 4,
             .size = 5,
             .block = 3,
-            .checksum = {0xee, 0xee, 0x05, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x05, 0, 0, 0, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1206,7 +1208,7 @@ testRun(void)
             .offset = 0,
             .size = 1,
             .block = 0,
-            .checksum = {0xee, 0xee, 0x06, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x06, 0, 0, 0, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1217,7 +1219,7 @@ testRun(void)
             .offset = 4,
             .size = 5,
             .block = 5,
-            .checksum = {0xee, 0xee, 0x07, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x07, 0, 0, 0, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1228,7 +1230,7 @@ testRun(void)
             .offset = 9,
             .size = 6,
             .block = 0,
-            .checksum = {0xee, 0xee, 0x08, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff},
+            .checksum = {0xee, 0xee, 0x08, 0, 0, 0, 0xff, 0xff},
         };
 
         TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
@@ -1238,7 +1240,7 @@ testRun(void)
 
         buffer = bufNew(256);
         write = ioBufferWriteNewOpen(buffer);
-        TEST_RESULT_VOID(blockMapWrite(blockMap, write, false), "save");
+        TEST_RESULT_VOID(blockMapWrite(blockMap, write, false, 8), "save");
         ioWriteClose(write);
 
         TEST_RESULT_STR_Z(
@@ -1248,35 +1250,35 @@ testRun(void)
             "00"                                        // reference 0
             "08"                                        // size 4
             "00"                                        // block 0
-            "eeee01000000000000000000000000000000ffff"  // checksum
+            "eeee01000000ffff"                          // checksum
 
             "03"                                        // block 1
-            "eeee02000000000000000000000000000000ffff"  // checksum
+            "eeee02000000ffff"                          // checksum
 
             "05"                                        // size 5
             "01"                                        // block 0
-            "eeee03000000000000000000000000000000ffff"  // checksum
+            "eeee03000000ffff"                          // checksum
 
             "08"                                        // reference 1
             "f902"                                      // size 99
             "01"                                        // block 0
-            "eeee04000000000000000000000000000000ffff"  // checksum
+            "eeee04000000ffff"                          // checksum
 
             "06"                                        // reference 0
             "07"                                        // block 3
-            "eeee05000000000000000000000000000000ffff"  // checksum
+            "eeee05000000ffff"                          // checksum
 
             "10"                                        // reference 2
             "0f"                                        // size 1
             "01"                                        // block 0
-            "eeee06000000000000000000000000000000ffff"  // checksum
+            "eeee06000000ffff"                          // checksum
 
             "03"                                        // reference 0
             "05"                                        // size 5
-            "eeee07000000000000000000000000000000ffff"  // checksum
+            "eeee07000000ffff"                          // checksum
             "05"                                        // size 6
             "01"                                        // block
-            "eeee08000000000000000000000000000000ffff", // checksum
+            "eeee08000000ffff",                         // checksum
             "compare");
 
         // -------------------------------------------------------------------------------------------------------------------------
@@ -1284,7 +1286,7 @@ testRun(void)
 
         bufferCompare = bufNew(256);
         write = ioBufferWriteNewOpen(bufferCompare);
-        TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer)), write, false), "read and save");
+        TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer), 8), write, false, 8), "read and save");
         ioWriteClose(write);
 
         TEST_RESULT_STR(strNewEncode(encodingHex, bufferCompare), strNewEncode(encodingHex, buffer), "compare");
@@ -1293,7 +1295,7 @@ testRun(void)
         TEST_TITLE("unequal block delta");
 
         TEST_RESULT_STR_Z(
-            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer)), 8),
+            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer), 8), 8, 8),
             "read {reference: 2, bundleId: 0, offset: 0, size: 1}\n"
             "  super block {size: 1}\n"
             "    block {no: 0, offset: 40}\n"
@@ -1324,6 +1326,12 @@ testRun(void)
         TEST_ERROR(
             backupBlockIncrMapSize(cfgOptRepoBlockSizeMap, 0, STRDEF("5GiB")), OptionInvalidValueError,
             "'5GiB' is not valid for 'repo1-block-size-map' option");
+        TEST_ERROR(
+            backupBlockIncrMapChecksumSize(cfgOptRepoBlockChecksumSizeMap, 0, VARSTRDEF("Z")), OptionInvalidValueError,
+            "'Z' is not valid for 'repo1-block-checksum-size-map' option");
+        TEST_ERROR(
+            backupBlockIncrMapChecksumSize(cfgOptRepoBlockChecksumSizeMap, 0, VARSTRDEF("5")), OptionInvalidValueError,
+            "'5' is not valid for 'repo1-block-checksum-size-map' option");
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("full backup with zero block");
@@ -1334,7 +1342,8 @@ testRun(void)
         Buffer *destination = bufNew(256);
         IoWrite *write = ioBufferWriteNew(destination);
 
-        TEST_RESULT_VOID(ioFilterGroupAdd(ioWriteFilterGroup(write), blockIncrNew(3, 3, 0, 0, 0, NULL, NULL, NULL)), "block incr");
+        TEST_RESULT_VOID(
+            ioFilterGroupAdd(ioWriteFilterGroup(write), blockIncrNew(3, 3, 6, 0, 0, 0, NULL, NULL, NULL)), "block incr");
         TEST_RESULT_VOID(ioWriteOpen(write), "open");
         TEST_RESULT_VOID(ioWrite(write, source), "write");
         TEST_RESULT_VOID(ioWriteClose(write), "close");
@@ -1349,14 +1358,15 @@ testRun(void)
         destination = bufNew(256);
         write = ioBufferWriteNew(destination);
 
-        TEST_RESULT_VOID(ioFilterGroupAdd(ioWriteFilterGroup(write), blockIncrNew(3, 3, 0, 0, 0, NULL, NULL, NULL)), "block incr");
+        TEST_RESULT_VOID(
+            ioFilterGroupAdd(ioWriteFilterGroup(write), blockIncrNew(3, 3, 8, 0, 0, 0, NULL, NULL, NULL)), "block incr");
         TEST_RESULT_VOID(ioWriteOpen(write), "open");
         TEST_RESULT_VOID(ioWrite(write, source), "write");
         TEST_RESULT_VOID(ioWriteClose(write), "close");
 
         uint64_t mapSize;
         TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size");
-        TEST_RESULT_UINT(mapSize, 23, "map size");
+        TEST_RESULT_UINT(mapSize, 11, "map size");
 
         TEST_RESULT_STR_Z(
             strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
@@ -1366,7 +1376,7 @@ testRun(void)
         const Buffer *map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
 
         TEST_RESULT_STR_Z(
-            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map)), 3),
+            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
             "read {reference: 0, bundleId: 0, offset: 0, size: 7}\n"
             "  super block {size: 7}\n"
             "    block {no: 0, offset: 0}\n",
@@ -1381,14 +1391,15 @@ testRun(void)
 
         TEST_RESULT_VOID(
             ioFilterGroupAdd(
-                ioWriteFilterGroup(write), blockIncrNewPack(ioFilterParamList(blockIncrNew(5, 3, 2, 4, 5, NULL, NULL, NULL)))),
+                ioWriteFilterGroup(write),
+                blockIncrNewPack(ioFilterParamList(blockIncrNew(5, 3, 8, 2, 4, 5, NULL, NULL, NULL)))),
             "block incr");
         TEST_RESULT_VOID(ioWriteOpen(write), "open");
         TEST_RESULT_VOID(ioWrite(write, source), "write");
         TEST_RESULT_VOID(ioWriteClose(write), "close");
 
         TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size");
-        TEST_RESULT_UINT(mapSize, 67, "map size");
+        TEST_RESULT_UINT(mapSize, 31, "map size");
 
         TEST_RESULT_STR_Z(
             strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
@@ -1400,7 +1411,7 @@ testRun(void)
         map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
 
         TEST_RESULT_STR_Z(
-            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map)), 3),
+            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
             "read {reference: 2, bundleId: 4, offset: 5, size: 27}\n"
             "  super block {size: 9}\n"
             "    block {no: 0, offset: 0}\n"
@@ -1421,14 +1432,14 @@ testRun(void)
 
         TEST_RESULT_VOID(
             ioFilterGroupAdd(
-                ioWriteFilterGroup(write), blockIncrNewPack(ioFilterParamList(blockIncrNew(3, 3, 3, 0, 0, map, NULL, NULL)))),
+                ioWriteFilterGroup(write), blockIncrNewPack(ioFilterParamList(blockIncrNew(3, 3, 8, 3, 0, 0, map, NULL, NULL)))),
             "block incr");
         TEST_RESULT_VOID(ioWriteOpen(write), "open");
         TEST_RESULT_VOID(ioWrite(write, source), "write");
         TEST_RESULT_VOID(ioWriteClose(write), "close");
 
         TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size");
-        TEST_RESULT_UINT(mapSize, 90, "map size");
+        TEST_RESULT_UINT(mapSize, 42, "map size");
 
         TEST_RESULT_STR_Z(
             strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
@@ -1439,7 +1450,7 @@ testRun(void)
         map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
 
         TEST_RESULT_STR_Z(
-            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map)), 3),
+            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
             "read {reference: 3, bundleId: 0, offset: 0, size: 13}\n"
             "  super block {size: 8}\n"
             "    block {no: 0, offset: 0}\n"
@@ -1463,14 +1474,14 @@ testRun(void)
 
         TEST_RESULT_VOID(
             ioFilterGroupAdd(
-                ioWriteFilterGroup(write), blockIncrNewPack(ioFilterParamList(blockIncrNew(6, 3, 2, 4, 5, NULL, NULL, NULL)))),
+                ioWriteFilterGroup(write), blockIncrNewPack(ioFilterParamList(blockIncrNew(6, 3, 8, 2, 4, 5, NULL, NULL, NULL)))),
             "block incr");
         TEST_RESULT_VOID(ioWriteOpen(write), "open");
         TEST_RESULT_VOID(ioWrite(write, source), "write");
         TEST_RESULT_VOID(ioWriteClose(write), "close");
 
         TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size");
-        TEST_RESULT_UINT(mapSize, 69, "map size");
+        TEST_RESULT_UINT(mapSize, 33, "map size");
 
         TEST_RESULT_STR_Z(
             strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
@@ -1482,7 +1493,7 @@ testRun(void)
         map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
 
         TEST_RESULT_STR_Z(
-            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map)), 3),
+            testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
             "read {reference: 2, bundleId: 4, offset: 5, size: 24}\n"
             "  super block {size: 15}\n"
             "    block {no: 0, offset: 0}\n"
@@ -1498,7 +1509,7 @@ testRun(void)
             blockIncrNewPack(
                 ioFilterParamList(
                     blockIncrNew(
-                        3, 3, 2, 4, 5, NULL, compressFilterP(compressTypeGz, 1, .raw = true),
+                        3, 3, 8, 2, 4, 5, NULL, compressFilterP(compressTypeGz, 1, .raw = true),
                         cipherBlockNewP(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS), .raw = true)))),
             "block incr pack");
     }
@@ -3973,7 +3984,7 @@ testRun(void)
                 "P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/0, 16.0KB, [PCT]) checksum [SHA1]\n"
                 "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16383, 8KB, [PCT]) checksum [SHA1]\n"
                 "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24575, 16KB, [PCT]) checksum [SHA1]\n"
-                "P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (bundle 1/41040, 2B, [PCT]) checksum [SHA1]\n"
+                "P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (bundle 1/40998, 2B, [PCT]) checksum [SHA1]\n"
                 "P00   INFO: execute non-exclusive backup stop and wait for all WAL segments to archive\n"
                 "P00   INFO: backup stop archive = 0000000105DBF06000000001, lsn = 5dbf060/300000\n"
                 "P00 DETAIL: wrote 'backup_label' file returned from backup stop function\n"
@@ -4003,9 +4014,9 @@ testRun(void)
                 ",\"timestamp\":1572800000}\n"
                 "pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
                 ",\"timestamp\":1572800002}\n"
-                "pg_data/block-incr-grow={\"bi\":1,\"bim\":68,\"checksum\":\"ebdd38b69cd5b9f2d00d273c981e16960fbbb4f7\""
+                "pg_data/block-incr-grow={\"bi\":1,\"bim\":26,\"checksum\":\"ebdd38b69cd5b9f2d00d273c981e16960fbbb4f7\""
                 ",\"size\":24576,\"timestamp\":1572800000}\n"
-                "pg_data/block-incr-shrink={\"bi\":1,\"bim\":72,\"checksum\":\"ce5f8864058b1bb274244b512cb9641355987134\""
+                "pg_data/block-incr-shrink={\"bi\":1,\"bim\":30,\"checksum\":\"ce5f8864058b1bb274244b512cb9641355987134\""
                 ",\"size\":16385,\"timestamp\":1572800000}\n"
                 "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1572800000}\n"
                 "pg_data/grow-to-block-incr={\"checksum\":\"f5a5c308cf5fcb52bccebe2365f8ed56acbcc41d\",\"size\":16383"
@@ -4083,8 +4094,8 @@ testRun(void)
                 "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-larger (1.4MB, [PCT]) checksum [SHA1]\n"
                 "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-grow (128KB, [PCT]) checksum [SHA1]\n"
                 "P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/0, 16KB, [PCT]) checksum [SHA1]\n"
-                "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16462, 8KB, [PCT]) checksum [SHA1]\n"
-                "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24654, 16.0KB, [PCT]) checksum [SHA1]\n"
+                "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16420, 8KB, [PCT]) checksum [SHA1]\n"
+                "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24612, 16.0KB, [PCT]) checksum [SHA1]\n"
                 "P00 DETAIL: reference pg_data/PG_VERSION to 20191103-165320F\n"
                 "P00   INFO: execute non-exclusive backup stop and wait for all WAL segments to archive\n"
                 "P00   INFO: backup stop archive = 0000000105DC213000000001, lsn = 5dc2130/300000\n"
@@ -4115,14 +4126,14 @@ testRun(void)
                 ",\"size\":2,\"timestamp\":1572800000}\n"
                 "pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
                 ",\"timestamp\":1573000002}\n"
-                "pg_data/block-incr-grow={\"bi\":1,\"bim\":347,\"checksum\":\"1ddde8db92dd9019be0819ae4f9ad9cea2fae399\""
+                "pg_data/block-incr-grow={\"bi\":1,\"bim\":123,\"checksum\":\"1ddde8db92dd9019be0819ae4f9ad9cea2fae399\""
                 ",\"size\":131072,\"timestamp\":1573000000}\n"
-                "pg_data/block-incr-larger={\"bi\":8,\"bim\":493,\"checksum\":\"eec53a6da79c00b3c658a7e09f44b3e9efefd960\","
-                "\"size\":1507328,\"timestamp\":1573000000}\n"
+                "pg_data/block-incr-larger={\"bi\":8,\"bic\":7,\"bim\":194"
+                ",\"checksum\":\"eec53a6da79c00b3c658a7e09f44b3e9efefd960\",\"size\":1507328,\"timestamp\":1573000000}\n"
                 "pg_data/block-incr-shrink={\"checksum\":\"1c6a17f67562d8b3f64f1b5f2ee592a4c2809b3b\",\"size\":16383"
                 ",\"timestamp\":1573000000}\n"
                 "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1573000000}\n"
-                "pg_data/grow-to-block-incr={\"bi\":1,\"bim\":69,\"checksum\":\"4f560611d9dc9212432970e5c4bec15d876c226e\","
+                "pg_data/grow-to-block-incr={\"bi\":1,\"bim\":27,\"checksum\":\"4f560611d9dc9212432970e5c4bec15d876c226e\","
                 "\"size\":16385,\"timestamp\":1573000000}\n"
                 "pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19"
                 ",\"timestamp\":1573000002}\n"
@@ -4227,7 +4238,7 @@ testRun(void)
                 ",\"timestamp\":1572800000}\n"
                 "pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
                 ",\"timestamp\":1573200002}\n"
-                "pg_data/block-incr-grow={\"bi\":1,\"bim\":88,\"checksum\":\"ebdd38b69cd5b9f2d00d273c981e16960fbbb4f7\","
+                "pg_data/block-incr-grow={\"bi\":1,\"bim\":40,\"checksum\":\"ebdd38b69cd5b9f2d00d273c981e16960fbbb4f7\","
                 "\"size\":24576,\"timestamp\":1573200000}\n"
                 "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1573200000}\n"
                 "pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19"
@@ -4257,6 +4268,8 @@ testRun(void)
             hrnCfgArgRawBool(argList, cfgOptRepoBlock, true);
             hrnCfgArgRawZ(argList, cfgOptRepoCipherType, "aes-256-cbc");
             hrnCfgArgRawZ(argList, cfgOptRepoBlockAgeMap, "1=2");
+            hrnCfgArgRawZ(argList, cfgOptRepoBlockChecksumSizeMap, "16KiB=16");
+            hrnCfgArgRawZ(argList, cfgOptRepoBlockChecksumSizeMap, "8KiB=12");
             hrnCfgEnvRawZ(cfgOptRepoCipherPass, TEST_CIPHER_PASS);
             HRN_CFG_LOAD(cfgCmdBackup, argList);
 
@@ -4319,9 +4332,9 @@ testRun(void)
                 ",\"size\":2,\"timestamp\":1572800000}\n"
                 "pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
                 ",\"timestamp\":1573400002}\n"
-                "pg_data/block-age-multiplier={\"bi\":2,\"bim\":56,\"checksum\":\"5188431849b4613152fd7bdba6a3ff0a4fd6424b\""
-                ",\"size\":32768,\"timestamp\":1573313600}\n"
-                "pg_data/block-incr-grow={\"bi\":1,\"bim\":152,\"checksum\":\"bd4716c88f38d2540e3024b54308b0b95f34a0cc\""
+                "pg_data/block-age-multiplier={\"bi\":2,\"bic\":16,\"bim\":56"
+                ",\"checksum\":\"5188431849b4613152fd7bdba6a3ff0a4fd6424b\",\"size\":32768,\"timestamp\":1573313600}\n"
+                "pg_data/block-incr-grow={\"bi\":1,\"bim\":72,\"checksum\":\"bd4716c88f38d2540e3024b54308b0b95f34a0cc\""
                 ",\"size\":49152,\"timestamp\":1573400000}\n"
                 "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1573400000}\n"
                 "pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19"

test/src/module/command/restoreTest.c

@@ -164,7 +164,7 @@ testRun(void)
 
         Buffer *output = bufNew(0);
         IoWrite *write = ioBufferWriteNew(output);
-        ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3));
+        ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3, 8));
         ioWriteOpen(write);
 
         TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("ABCDEF")), "write");
@@ -173,17 +173,19 @@ testRun(void)
 
         TEST_RESULT_STR_Z(
             strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_HASH_FILTER_TYPE))),
-            "3c01bdbb26f358bab27f267924aa2c9a03fcfdb8"
-            "6dae29c06c5f04601445c493156d10fe1be23b6d"
-            "3c01bdbb26f358bab27f267924aa2c9a03fcfdb8",
+            "9e947f00ecd6acb2"
+            "cb221327e5a387af"
+            "9e947f00ecd6acb2",
             "block hash list");
 
+        ioWriteFree(write);
+
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("buffer smaller than block and remainder");
 
         output = bufNew(0);
         write = ioBufferWriteNew(output);
-        ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3));
+        ioFilterGroupAdd(ioWriteFilterGroup(write), blockHashNew(3, 8));
         ioWriteOpen(write);
 
         TEST_RESULT_VOID(ioWrite(write, BUFSTRDEF("DE")), "write");
@@ -195,11 +197,13 @@ testRun(void)
 
         TEST_RESULT_STR_Z(
             strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_HASH_FILTER_TYPE))),
-            "6dae29c06c5f04601445c493156d10fe1be23b6d"
-            "3c01bdbb26f358bab27f267924aa2c9a03fcfdb8"
-            "3c01bdbb26f358bab27f267924aa2c9a03fcfdb8"
-            "c032adc1ff629c9b66f22749ad667e6beadf144b",
+            "cb221327e5a387af"
+            "9e947f00ecd6acb2"
+            "9e947f00ecd6acb2"
+            "92c3453969207870",
             "block hash list");
+
+        ioWriteFree(write);
     }
 
     // *****************************************************************************************************************************
@@ -2661,7 +2665,7 @@ testRun(void)
             bufUsedSet(fileBuffer, bufSize(fileBuffer));
 
             IoWrite *write = storageWriteIo(storageNewWriteP(storageRepoWrite(), STRDEF(TEST_REPO_PATH "base/1/bi-no-ref.pgbi")));
-            ioFilterGroupAdd(ioWriteFilterGroup(write), blockIncrNew(8192, 8192, 3, 0, 0, NULL, NULL, NULL));
+            ioFilterGroupAdd(ioWriteFilterGroup(write), blockIncrNew(8192, 8192, 11, 3, 0, 0, NULL, NULL, NULL));
             ioFilterGroupAdd(ioWriteFilterGroup(write), ioSizeNew());
 
             ioWriteOpen(write);
@@ -2673,7 +2677,7 @@ testRun(void)
 
             HRN_MANIFEST_FILE_ADD(
                 manifest, .name = TEST_PGDATA "base/1/bi-no-ref", .size = bufUsed(fileBuffer), .sizeRepo = repoSize,
-                .blockIncrSize = 8192, .blockIncrMapSize = blockIncrMapSize, .timestamp = 1482182860,
+                .blockIncrSize = 8192, .blockIncrChecksumSize = 11, .blockIncrMapSize = blockIncrMapSize, .timestamp = 1482182860,
                 .checksumSha1 = "953cdcc904c5d4135d96fc0833f121bf3033c74c");
 
             // Block incremental with a broken reference to show that unneeded references will not be used
@@ -2683,7 +2687,7 @@ testRun(void)
 
             Buffer *fileUnusedMap = bufNew(0);
             write = ioBufferWriteNew(fileUnusedMap);
-            ioFilterGroupAdd(ioWriteFilterGroup(write), blockIncrNew(8192, 8192, 0, 0, 0, NULL, NULL, NULL));
+            ioFilterGroupAdd(ioWriteFilterGroup(write), blockIncrNew(8192, 8192, 11, 0, 0, 0, NULL, NULL, NULL));
 
             ioWriteOpen(write);
             ioWrite(write, fileUnused);
@@ -2702,8 +2706,8 @@ testRun(void)
             ioFilterGroupAdd(
                 ioWriteFilterGroup(write),
                 blockIncrNew(
-                    8192, 8192, 3, 0, 0, BUF(bufPtr(fileUnusedMap) + bufUsed(fileUnusedMap) - fileUnusedMapSize, fileUnusedMapSize),
-                    NULL, NULL));
+                    8192, 8192, 11, 3, 0, 0,
+                    BUF(bufPtr(fileUnusedMap) + bufUsed(fileUnusedMap) - fileUnusedMapSize, fileUnusedMapSize), NULL, NULL));
             ioFilterGroupAdd(ioWriteFilterGroup(write), ioSizeNew());
 
             ioWriteOpen(write);
@@ -2717,7 +2721,7 @@ testRun(void)
 
             HRN_MANIFEST_FILE_ADD(
                 manifest, .name = TEST_PGDATA "base/1/bi-unused-ref", .size = bufUsed(fileUsed), .sizeRepo = fileUsedRepoSize,
-                .blockIncrSize = 8192, .blockIncrMapSize = fileUsedMapSize, .timestamp = 1482182860,
+                .blockIncrSize = 8192, .blockIncrChecksumSize = 11, .blockIncrMapSize = fileUsedMapSize, .timestamp = 1482182860,
                 .checksumSha1 = "febd680181d4cd315dce942348862c25fbd731f3");
 
             memset(bufPtr(fileUnused) + (8192 * 4), 3, 8192);

test/src/module/common/cryptoTest.c

@@ -1,6 +1,7 @@
 /***********************************************************************************************************************************
 Test Block Cipher
 ***********************************************************************************************************************************/
+#include "common/io/bufferRead.h"
 #include "common/io/filter/filter.h"
 #include "common/io/io.h"
 #include "common/type/json.h"
@@ -404,5 +405,66 @@ testRun(void)
             "    check hmac");
     }
 
+    // *****************************************************************************************************************************
+    if (testBegin("XxHash"))
+    {
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("xxHashOne");
+
+        TEST_RESULT_STR_Z(
+            strNewEncode(encodingHex, xxHashOne(16, BUFSTRDEF(""))), "99aa06d3014798d86001c324468d497f", "check empty hash 16");
+        TEST_RESULT_STR_Z(
+            strNewEncode(encodingHex, xxHashOne(16, BUFSTRDEF("12345\n"))), "1a3e11127b8856b804f0f99dc9fa4b56",
+            "check small hash 16");
+
+        TEST_RESULT_STR_Z(strNewEncode(encodingHex, xxHashOne(5, BUFSTRDEF(""))), "99aa06d301", "check empty hash 5");
+        TEST_RESULT_STR_Z(strNewEncode(encodingHex, xxHashOne(5, BUFSTRDEF("12345\n"))), "1a3e11127b", "check small hash 5");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("XxHash");
+
+        ioBufferSizeSet(2);
+
+        const Buffer *message = BUFSTRDEF("");
+        IoRead *read = ioBufferReadNew(message);
+        ioFilterGroupAdd(ioReadFilterGroup(read), xxHashNew(16));
+
+        ioReadDrain(read);
+
+        TEST_RESULT_STR_Z(
+            strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), XX_HASH_FILTER_TYPE))),
+            "99aa06d3014798d86001c324468d497f", "check empty hash 16");
+
+        message = BUFSTRDEF("12345\n");
+        read = ioBufferReadNew(message);
+        ioFilterGroupAdd(ioReadFilterGroup(read), xxHashNew(16));
+
+        ioReadDrain(read);
+
+        TEST_RESULT_STR_Z(
+            strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), XX_HASH_FILTER_TYPE))),
+            "1a3e11127b8856b804f0f99dc9fa4b56", "check small hash 16");
+
+        message = BUFSTRDEF("");
+        read = ioBufferReadNew(message);
+        ioFilterGroupAdd(ioReadFilterGroup(read), xxHashNew(5));
+
+        ioReadDrain(read);
+
+        TEST_RESULT_STR_Z(
+            strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), XX_HASH_FILTER_TYPE))),
+            "99aa06d301", "check empty hash 5");
+
+        message = BUFSTRDEF("12345\n");
+        read = ioBufferReadNew(message);
+        ioFilterGroupAdd(ioReadFilterGroup(read), xxHashNew(5));
+
+        ioReadDrain(read);
+
+        TEST_RESULT_STR_Z(
+            strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), XX_HASH_FILTER_TYPE))),
+            "1a3e11127b", "check small hash 5");
+    }
+
     FUNCTION_HARNESS_RETURN_VOID();
 }

test/src/module/info/manifestTest.c

@@ -774,12 +774,21 @@ testRun(void)
             {.fileAge = 7 * 86400, .blockMultiplier = 2},
         };
 
+        static const ManifestBlockIncrChecksumSizeMap manifestBlockIncrChecksumSizeMap[] =
+        {
+            {.blockSize = 512 * 1024, .checksumSize = BLOCK_INCR_CHECKSUM_SIZE_MIN + 3},
+            {.blockSize = 128 * 1024, .checksumSize = BLOCK_INCR_CHECKSUM_SIZE_MIN + 2},
+            {.blockSize = 32 * 1024, .checksumSize = BLOCK_INCR_CHECKSUM_SIZE_MIN + 1},
+        };
+
         static const ManifestBlockIncrMap manifestBuildBlockIncrMap =
         {
             .sizeMap = manifestBlockIncrSizeMap,
             .sizeMapSize = LENGTH_OF(manifestBlockIncrSizeMap),
             .ageMap = manifestBlockIncrAgeMap,
             .ageMapSize = LENGTH_OF(manifestBlockIncrAgeMap),
+            .checksumSizeMap = manifestBlockIncrChecksumSizeMap,
+            .checksumSizeMapSize = LENGTH_OF(manifestBlockIncrChecksumSizeMap),
         };
 
         // pg_wal not ignored
@@ -807,8 +816,8 @@ testRun(void)
                     "pg_data/postgresql.conf={\"file\":\"postgresql.conf\",\"path\":\"../config\",\"type\":\"link\"}\n"
                     "\n"
                     "[target:file]\n"
-                    "pg_data/128k={\"bi\":16,\"size\":131072,\"timestamp\":1570000000}\n"
-                    "pg_data/128k-1week={\"bi\":32,\"size\":131072,\"timestamp\":1569395200}\n"
+                    "pg_data/128k={\"bi\":16,\"bic\":8,\"size\":131072,\"timestamp\":1570000000}\n"
+                    "pg_data/128k-1week={\"bi\":32,\"bic\":8,\"size\":131072,\"timestamp\":1569395200}\n"
                     "pg_data/128k-4week={\"size\":131072,\"timestamp\":1567580800}\n"
                     "pg_data/PG_VERSION={\"size\":3,\"timestamp\":1565282100}\n"
                     "pg_data/base/1/555_init={\"size\":0,\"timestamp\":1565282114}\n"
@@ -1345,7 +1354,7 @@ testRun(void)
         // Prior file was not block incr but current file is
         HRN_MANIFEST_FILE_ADD(
             manifest, .name = MANIFEST_TARGET_PGDATA "/block-incr-add", .copy = true, .size = 6, .sizeRepo = 6,
-            .blockIncrSize = 8192, .timestamp = 1482182861, .group = "test", .user = "test");
+            .blockIncrSize = 8192, .blockIncrChecksumSize = 6, .timestamp = 1482182861, .group = "test", .user = "test");
         HRN_MANIFEST_FILE_ADD(
             manifestPrior, .name = MANIFEST_TARGET_PGDATA "/block-incr-add", .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
             .checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa");
@@ -1361,10 +1370,11 @@ testRun(void)
         // Prior file has different block incr size
         HRN_MANIFEST_FILE_ADD(
             manifest, .name = MANIFEST_TARGET_PGDATA "/block-incr-keep-size", .copy = true, .size = 6, .sizeRepo = 6,
-            .blockIncrSize = 16384, .timestamp = 1482182861, .group = "test", .user = "test");
+            .blockIncrSize = 16384, .blockIncrChecksumSize = 6, .timestamp = 1482182861, .group = "test", .user = "test");
         HRN_MANIFEST_FILE_ADD(
             manifestPrior, .name = MANIFEST_TARGET_PGDATA "/block-incr-keep-size", .size = 4, .sizeRepo = 4, .blockIncrSize = 8192,
-            .blockIncrMapSize = 31, .timestamp = 1482182860, .checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa");
+            .blockIncrChecksumSize = 6, .blockIncrMapSize = 31, .timestamp = 1482182860,
+            .checksumSha1 = "ddddddddddbbbbbbbbbbccccccccccaaaaaaaaaa");
 
         TEST_RESULT_VOID(
             manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, STRDEF("000000030000000300000003")), "incremental manifest");
@@ -1570,10 +1580,11 @@ testRun(void)
                 ",\"timestamp\":1565282114}\n"                                                                                     \
             "pg_data/base/16384/PG_VERSION={\"bni\":1,\"bno\":1,\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\""         \
                 ",\"group\":\"group2\",\"size\":4,\"timestamp\":1565282115,\"user\":false}\n"                                      \
-            "pg_data/base/32768/33000={\"checksum\":\"7a16d165e4775f7c92e8cdf60c0af57313f0bf90\",\"checksum-page\":true"           \
-                ",\"reference\":\"20190818-084502F\",\"size\":1073741824,\"timestamp\":1565282116}\n"                              \
-            "pg_data/base/32768/33000.32767={\"bi\":3,\"bim\":96,\"checksum\":\"6e99b589e550e68e934fd235ccba59fe5b592a9e\","       \
-                "\"checksum-page\":true,\"reference\":\"20190818-084502F\",\"size\":32768,\"timestamp\":1565282114}\n"             \
+            "pg_data/base/32768/33000={\"bi\":4,\"bim\":99,\"checksum\":\"7a16d165e4775f7c92e8cdf60c0af57313f0bf90\""              \
+                ",\"checksum-page\":true,\"reference\":\"20190818-084502F\",\"size\":1073741824,\"timestamp\":1565282116}\n"       \
+            "pg_data/base/32768/33000.32767={\"bi\":3,\"bic\":16,\"bim\":96"                                                       \
+                ",\"checksum\":\"6e99b589e550e68e934fd235ccba59fe5b592a9e\",\"checksum-page\":true"                                \
+                ",\"reference\":\"20190818-084502F\",\"size\":32768,\"timestamp\":1565282114}\n"                                   \
             "pg_data/postgresql.conf={\"size\":4457,\"timestamp\":1565282114}\n"                                                   \
             "pg_data/special-@#!$^&*()_+~`{}[]\\:;={\"mode\":\"0640\",\"size\":0,\"timestamp\":1565282120,\"user\":false}\n"