diff --git a/doc/xml/release.xml b/doc/xml/release.xml
index 6298fee1b..65c106dae 100644
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -54,6 +54,9 @@
+
+
+
diff --git a/src/command/backup/backup.c b/src/command/backup/backup.c
index 09fc3dd91..a2faabfbc 100644
--- a/src/command/backup/backup.c
+++ b/src/command/backup/backup.c
@@ -271,20 +271,19 @@ backupInit(const InfoBackup *infoBackup)
Build block incremental maps
***********************************************************************************************************************************/
// Size map. Block size is increased when the block map would be larger than a single block. The break can be calculated with this
-// formula: [block size in KiB] / (1024 / [block size in KiB] * [checksum size + 1]) * 1073741824.
+// formula: [block size in KiB] / (1024 / [block size in KiB] * [checksum size]) * 1073741824.
static const ManifestBlockIncrSizeMap manifestBlockIncrSizeMapDefault[] =
{
- {.fileSize = 968 * 1024 * 1024, .blockSize = 96 * 1024},
- {.fileSize = 800 * 1024 * 1024, .blockSize = 88 * 1024},
- {.fileSize = 648 * 1024 * 1024, .blockSize = 80 * 1024},
- {.fileSize = 512 * 1024 * 1024, .blockSize = 72 * 1024},
- {.fileSize = 392 * 1024 * 1024, .blockSize = 64 * 1024},
- {.fileSize = 288 * 1024 * 1024, .blockSize = 56 * 1024},
- {.fileSize = 200 * 1024 * 1024, .blockSize = 48 * 1024},
- {.fileSize = 128 * 1024 * 1024, .blockSize = 40 * 1024},
- {.fileSize = 84256 * 1024, .blockSize = 32 * 1024}, // These do not come out evenly because the map record size is 7
- {.fileSize = 37448 * 1024, .blockSize = 24 * 1024}, // instead of 8 in the rows above
- {.fileSize = 9360 * 1024, .blockSize = 16 * 1024},
+ {.fileSize = 914 * 1024 * 1024, .blockSize = 88 * 1024},
+ {.fileSize = 740 * 1024 * 1024, .blockSize = 80 * 1024},
+ {.fileSize = 585 * 1024 * 1024, .blockSize = 72 * 1024},
+ {.fileSize = 448 * 1024 * 1024, .blockSize = 64 * 1024},
+ {.fileSize = 329 * 1024 * 1024, .blockSize = 56 * 1024},
+ {.fileSize = 228 * 1024 * 1024, .blockSize = 48 * 1024},
+ {.fileSize = 146 * 1024 * 1024, .blockSize = 40 * 1024},
+ {.fileSize = 96 * 1024 * 1024, .blockSize = 32 * 1024},
+ {.fileSize = 43 * 1024 * 1024, .blockSize = 24 * 1024},
+ {.fileSize = 11 * 1024 * 1024, .blockSize = 16 * 1024},
{.fileSize = 16 * 1024, .blockSize = 8 * 1024},
};
diff --git a/src/command/backup/blockIncr.c b/src/command/backup/blockIncr.c
index 4ae7f3dd3..e4d6a53b0 100644
--- a/src/command/backup/blockIncr.c
+++ b/src/command/backup/blockIncr.c
@@ -174,15 +174,8 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output)
ioWriteOpen(this->blockOutWrite);
}
- // Write the block no as a delta of the prior block no. If the size of the last block is smaller than block
- // size then write the smaller block size.
- const uint64_t blockEncoded = bufUsed(this->block) < this->blockSize ? BLOCK_INCR_FLAG_SIZE : 0;
-
- ioWriteVarIntU64(
- this->blockOutWrite, blockEncoded | ((this->blockNo - this->blockNoLast) << BLOCK_INCR_BLOCK_SHIFT));
-
- if (blockEncoded & BLOCK_INCR_FLAG_SIZE)
- ioWriteVarIntU64(this->blockOutWrite, bufUsed(this->block));
+ // Write the block no as a delta of the prior block no
+ ioWriteVarIntU64(this->blockOutWrite, this->blockNo - this->blockNoLast);
// Copy block data through the filters
ioCopyP(ioBufferReadNewOpen(this->block), this->blockOutWrite);
@@ -193,6 +186,7 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output)
BlockMapItem blockMapItem =
{
.reference = this->reference,
+ .superBlockSize = this->superBlockSize,
.bundleId = this->bundleId,
.offset = this->blockOffset,
.block = this->superBlockNo,
@@ -225,22 +219,20 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output)
// Write the super block
if (this->blockOutWrite != NULL && (this->done || this->blockOutSize >= this->superBlockSize))
{
- // Explicitly terminate the block if all block sizes are equal. This is not required if the last block is smaller than
- // the block size.
- if (this->blockOutSize % this->blockSize == 0)
- ioWriteVarIntU64(this->blockOutWrite, 0);
-
// Close write
ioWriteClose(this->blockOutWrite);
- // Update size for items already added to the block map
+ // Update size and super block size for items already added to the block map
const uint64_t blockOutSize = pckReadU64P(
ioFilterGroupResultP(ioWriteFilterGroup(this->blockOutWrite), SIZE_FILTER_TYPE));
for (unsigned int blockMapIdx = 0; blockMapIdx < lstSize(this->blockOutList); blockMapIdx++)
{
- blockMapGet(
- this->blockMapOut, *(unsigned int *)lstGet(this->blockOutList, blockMapIdx))->size = blockOutSize;
+ BlockMapItem *const blockMapItem = blockMapGet(
+ this->blockMapOut, *(unsigned int *)lstGet(this->blockOutList, blockMapIdx));
+
+ blockMapItem->size = blockOutSize;
+ blockMapItem->superBlockSize = this->blockOutSize;
}
lstFree(this->blockOutList);
@@ -273,7 +265,7 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output)
// Write the map
ioWriteOpen(write);
- blockMapWrite(this->blockMapOut, write, this->blockSize == this->superBlockSize, this->checksumSize);
+ blockMapWrite(this->blockMapOut, write, this->blockSize, this->checksumSize);
ioWriteClose(write);
// Get total bytes written for the map
@@ -407,7 +399,7 @@ blockIncrNew(
*driver = (BlockIncr)
{
.memContext = memContextCurrent(),
- .superBlockSize = superBlockSize,
+ .superBlockSize = (superBlockSize / blockSize + (superBlockSize % blockSize == 0 ? 0 : 1)) * blockSize,
.blockSize = blockSize,
.checksumSize = checksumSize,
.reference = reference,
@@ -418,10 +410,6 @@ blockIncrNew(
.blockMapOut = blockMapNew(),
};
- // If there will be less than two blocks per super block then set them equal to save space in the map
- if (superBlockSize < blockSize * 2)
- driver->superBlockSize = blockSize;
-
// Duplicate compress filter
if (compress != NULL)
{
@@ -442,7 +430,7 @@ blockIncrNew(
MEM_CONTEXT_PRIOR_BEGIN()
{
- driver->blockMapPrior = blockMapNewRead(read, checksumSize);
+ driver->blockMapPrior = blockMapNewRead(read, blockSize, checksumSize);
}
MEM_CONTEXT_PRIOR_END();
}
diff --git a/src/command/backup/blockIncr.h b/src/command/backup/blockIncr.h
index 47dd5f79a..60255fcb7 100644
--- a/src/command/backup/blockIncr.h
+++ b/src/command/backup/blockIncr.h
@@ -20,9 +20,6 @@ consists of the following, compressed and encrypted as required:
when the block list is being read sequentially, e.g. during verification. If blocks are accessed from the map then the block
number is already known and the delta can be ignored.
-- If the block number above has BLOCK_INCR_FLAG_SIZE set then the block size is different from the default and must be read. This is
- because it is at the end of the file and so also indicates the end of the super block.
-
- Block data.
The super block list is followed by the block map, which is encrypted separately when required but not compressed. The return value
@@ -48,12 +45,6 @@ Filter type constant
***********************************************************************************************************************************/
#define BLOCK_INCR_FILTER_TYPE STRID5("blk-incr", 0x90dc9dad820)
-/***********************************************************************************************************************************
-Constants needed to read the block number in a super block
-***********************************************************************************************************************************/
-#define BLOCK_INCR_FLAG_SIZE 1 // Does the block have a different size than the default?
-#define BLOCK_INCR_BLOCK_SHIFT 1 // Shift required to get block number
-
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
diff --git a/src/command/backup/blockMap.c b/src/command/backup/blockMap.c
index 5a3e6c59c..24761e5da 100644
--- a/src/command/backup/blockMap.c
+++ b/src/command/backup/blockMap.c
@@ -40,22 +40,27 @@ References, super blocks, and blocks are encoded with a bit that indicates when
#define BLOCK_MAP_FLAG_CONTINUE 2 // Reference continues a prior super block
#define BLOCK_MAP_FLAG_CONTINUE_LAST 4 // Continued super block is last for the reference
#define BLOCK_MAP_REFERENCE_SHIFT 3 // Shift bits for reference
-#define BLOCK_MAP_SUPER_BLOCK_SHIFT 1 // Shift bits for super block
-#define BLOCK_MAP_BLOCK_SHIFT 1 // Shift bits for block
+#define BLOCK_MAP_FLAG_SUPER_BLOCK_SIZE_REMAINDER 1 // Remainder for super block size
+#define BLOCK_MAP_SUPER_BLOCK_SIZE_SHIFT 1 // Shift bits for super block size
+#define BLOCK_MAP_FLAG_SUPER_BLOCK_CHANGE 2 // The super block size has changed
+#define BLOCK_MAP_FLAG_SUPER_BLOCK_TOTAL_OFFSET 4 // Block total/offset when not complete
+#define BLOCK_MAP_SUPER_BLOCK_SHIFT 3 // Shift bits for super block
+#define BLOCK_MAP_FLAG_BLOCK_TOTAL_OFFSET 1 // Block total has an offset
+#define BLOCK_MAP_BLOCK_TOTAL_SHIFT 1 // Shift bits for block total
typedef enum
{
blockMapFlagVersion = 0, // Version (currently always 0)
- blockMapFlagEqual = 1, // Are blocks and super blocks equal size?
} BlockMapFlag;
// Stores current information about a reference to avoid needed to encode it again
typedef struct BlockMapReference
{
unsigned int reference; // Reference
+ uint64_t superBlockSize; // Super block size
uint64_t bundleId; // Bundle id
uint64_t offset; // Offset
- uint64_t size; // Super block size
+ uint64_t size; // Stored super block size (with compression, etc.)
uint64_t block; // Block no
} BlockMapReference;
@@ -80,7 +85,7 @@ lstComparatorBlockMapReference(const void *const blockMapRef1, const void *const
}
FN_EXTERN BlockMap *
-blockMapNewRead(IoRead *const map, size_t checksumSize)
+blockMapNewRead(IoRead *const map, const size_t blockSize, const size_t checksumSize)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(IO_READ, map);
@@ -88,11 +93,7 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
// Read flags. Currently the version flag must always be zero. This may be used in the future to indicate if the map version
// has changed.
- const uint64_t flags = ioReadVarIntU64(map);
- CHECK(FormatError, (flags & (1 << blockMapFlagVersion)) == 0, "block map version must be zero");
-
- // Are the super block and block size equal?
- const bool blockEqual = flags & (1 << blockMapFlagEqual);
+ CHECK(FormatError, (ioReadVarIntU64(map) & (1 << blockMapFlagVersion)) == 0, "block map version must be zero");
// Read all references in packed format
BlockMap *const this = blockMapNew();
@@ -119,10 +120,14 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
if (referenceEncoded & BLOCK_MAP_FLAG_OFFSET)
blockMapItem.offset = ioReadVarIntU64(map);
+ // Default super block size
+ blockMapItem.superBlockSize = blockSize;
+
// Add reference to list
BlockMapReference referenceDataAdd =
{
.reference = blockMapItem.reference,
+ .superBlockSize = blockMapItem.superBlockSize,
.bundleId = blockMapItem.bundleId,
.offset = blockMapItem.offset,
};
@@ -132,13 +137,12 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
// Else this reference has been read before
else
{
+ blockMapItem.superBlockSize = referenceData->superBlockSize;
blockMapItem.bundleId = referenceData->bundleId;
// If the reference is continued use the prior offset and size values
if (referenceEncoded & BLOCK_MAP_FLAG_CONTINUE)
{
- ASSERT(!blockEqual);
-
blockMapItem.offset = referenceData->offset;
blockMapItem.size = referenceData->size;
referenceContinue = true;
@@ -168,6 +172,7 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
if (referenceEncoded & BLOCK_MAP_FLAG_CONTINUE_LAST)
superBlockEncoded = BLOCK_MAP_FLAG_LAST;
+ superBlockEncoded |= BLOCK_MAP_FLAG_SUPER_BLOCK_TOTAL_OFFSET;
referenceContinue = false;
}
// Else read the super block size for the reference
@@ -175,14 +180,24 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
{
superBlockEncoded = ioReadVarIntU64(map);
- // If this is the first size read then just read the size
- if (sizeLast == 0)
+ // If this is the first size read then just read the size. Otherwise read the difference from the prior size and
+ // add sizeLast.
+ blockMapItem.size = superBlockEncoded >> BLOCK_MAP_SUPER_BLOCK_SHIFT;
+
+ if (sizeLast != 0)
+ blockMapItem.size = (uint64_t)(cvtInt64FromZigZag(blockMapItem.size) + sizeLast);
+
+ // If the super block size has changed then read it
+ if (superBlockEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_CHANGE)
{
- blockMapItem.size = superBlockEncoded >> BLOCK_MAP_SUPER_BLOCK_SHIFT;
+ const uint64_t superBlockSizeEncoded = ioReadVarIntU64(map);
+ blockMapItem.superBlockSize = (superBlockSizeEncoded >> BLOCK_MAP_SUPER_BLOCK_SIZE_SHIFT) * blockSize;
+
+ if (superBlockSizeEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_SIZE_REMAINDER)
+ blockMapItem.superBlockSize += ioReadVarIntU64(map);
+
+ referenceData->superBlockSize = blockMapItem.superBlockSize;
}
- // Else read the difference from the prior size and apply to sizeLast
- else
- blockMapItem.size = (uint64_t)(cvtInt64FromZigZag(superBlockEncoded >> BLOCK_MAP_SUPER_BLOCK_SHIFT) + sizeLast);
// Set offset, size, and block for the super block
if (superBlockFirst)
@@ -198,19 +213,26 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
sizeLast = (int64_t)blockMapItem.size;
superBlockFirst = false;
- // Read all blocks in the current super block in packed format
- do
- {
- // Block no only needs to be read when super block size does not equal block size
- uint64_t blockEncoded = BLOCK_MAP_FLAG_LAST;
+ // Read or calculate block total
+ uint64_t blockTotal;
- if (!blockEqual)
- {
- // Read block no as the difference from the prior block
- blockEncoded = ioReadVarIntU64(map);
- blockMapItem.block = (blockEncoded >> BLOCK_MAP_BLOCK_SHIFT) + referenceData->block;
- referenceData->block = blockMapItem.block;
- }
+ if (superBlockEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_TOTAL_OFFSET)
+ {
+ const uint64_t blockTotalEncoded = ioReadVarIntU64(map);
+ blockTotal = (blockTotalEncoded >> BLOCK_MAP_BLOCK_TOTAL_SHIFT) + 1;
+
+ // Offset block no from expected
+ if (blockTotalEncoded & BLOCK_MAP_FLAG_BLOCK_TOTAL_OFFSET)
+ referenceData->block += ioReadVarIntU64(map);
+ }
+ else
+ blockTotal = blockMapItem.superBlockSize / blockSize + (blockMapItem.superBlockSize % blockSize == 0 ? 0 : 1);
+
+ // Read checksums
+ for (uint64_t blockIdx = 0; blockIdx < blockTotal; blockIdx++)
+ {
+ // Set block no
+ blockMapItem.block = referenceData->block + blockIdx;
// Read checksum
bufUsedZero(checksum);
@@ -219,12 +241,10 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
// Add to block list
lstAdd((List *)this, &blockMapItem);
-
- // Break when this is the last block in the super block
- if (blockEncoded & BLOCK_MAP_FLAG_LAST)
- break;
}
- while (true);
+
+ // Update block in reference with all blocks read
+ referenceData->block += blockTotal;
// Update offset with the super block size
blockMapItem.offset += blockMapItem.size;
@@ -249,21 +269,22 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
/**********************************************************************************************************************************/
FN_EXTERN void
-blockMapWrite(const BlockMap *const this, IoWrite *const output, const bool blockEqual, const size_t checksumSize)
+blockMapWrite(const BlockMap *const this, IoWrite *const output, const size_t blockSize, const size_t checksumSize)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BLOCK_MAP, this);
FUNCTION_LOG_PARAM(IO_WRITE, output);
- FUNCTION_LOG_PARAM(BOOL, blockEqual);
+ FUNCTION_LOG_PARAM(SIZE, blockSize);
FUNCTION_LOG_PARAM(SIZE, checksumSize);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(blockMapSize(this) > 0);
+ ASSERT(blockSize > 0);
ASSERT(output != NULL);
- // Set flag to indicate that super block and block size are equal
- ioWriteVarIntU64(output, blockEqual ? 1 << blockMapFlagEqual : 0);
+ // Write flags
+ ioWriteVarIntU64(output, 0);
// Write all references in packed format
List *const refList = lstNewP(sizeof(BlockMapReference), .comparator = lstComparatorBlockMapReference);
@@ -317,6 +338,7 @@ blockMapWrite(const BlockMap *const this, IoWrite *const output, const bool bloc
const BlockMapReference referenceAdd =
{
.reference = reference->reference,
+ .superBlockSize = blockSize,
.bundleId = reference->bundleId,
.offset = reference->offset,
};
@@ -335,7 +357,7 @@ blockMapWrite(const BlockMap *const this, IoWrite *const output, const bool bloc
// the last one for the reference.
if (reference->offset == referenceData->offset)
{
- ASSERT(!blockEqual);
+ ASSERT(reference->superBlockSize == referenceData->superBlockSize);
referenceEncoded |= BLOCK_MAP_FLAG_CONTINUE;
referenceContinue = true;
@@ -374,9 +396,21 @@ blockMapWrite(const BlockMap *const this, IoWrite *const output, const bool bloc
}
}
+ // If block offset or total need to be stored then add the flag
+ const unsigned int blockTotal = superBlockIdx - blockIdx;
+ ASSERT(blockTotal > 0);
+
+ if (referenceContinue || superBlock->block != 0 ||
+ blockTotal != superBlock->superBlockSize / blockSize + (superBlock->superBlockSize % blockSize == 0 ? 0 : 1))
+ {
+ superBlockEncoded |= BLOCK_MAP_FLAG_SUPER_BLOCK_TOTAL_OFFSET;
+ }
+
// Write the continued reference now that we know if this will be the last super block in the reference
if (referenceContinue)
{
+ ASSERT(superBlock->superBlockSize == referenceData->superBlockSize);
+
if (superBlockEncoded & BLOCK_MAP_FLAG_LAST)
referenceEncoded |= BLOCK_MAP_FLAG_CONTINUE_LAST;
@@ -386,50 +420,76 @@ blockMapWrite(const BlockMap *const this, IoWrite *const output, const bool bloc
// Else write the super block size for the reference
else
{
- // If this is the first size written then just write the size
- if (sizeLast == 0)
- {
- ioWriteVarIntU64(output, superBlockEncoded | superBlock->size << BLOCK_MAP_SUPER_BLOCK_SHIFT);
- }
- // Else write the difference from the prior size. This depends on the expectation that the compressed size of
- // equal-sized blocks will be similar in order to be most efficient.
- else
- {
- ioWriteVarIntU64(
- output,
- superBlockEncoded | cvtInt64ToZigZag((int64_t)superBlock->size - sizeLast) << BLOCK_MAP_SUPER_BLOCK_SHIFT);
- }
-
// Set offset, size, and block for the super block
referenceData->offset = superBlock->offset;
referenceData->size = superBlock->size;
referenceData->block = 0;
+
+ // If the super block size has changed then add the flag
+ ASSERT(reference->superBlockSize > 0);
+
+ if (superBlock->superBlockSize != referenceData->superBlockSize)
+ superBlockEncoded |= BLOCK_MAP_FLAG_SUPER_BLOCK_CHANGE;
+
+ // If this is the first size written then just write the size. Otherwise write the difference from the prior size.
+ // This depends on the expectation that the compressed size of equal-sized blocks will be similar in order to be
+ // most efficient.
+ ioWriteVarIntU64(
+ output,
+ superBlockEncoded |
+ (sizeLast == 0 ? superBlock->size : cvtInt64ToZigZag((int64_t)superBlock->size - sizeLast)) <<
+ BLOCK_MAP_SUPER_BLOCK_SHIFT);
+
+ // If the super block size has changed then write it
+ if (superBlockEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_CHANGE)
+ {
+ const uint64_t superBlockSizeEncoded =
+ (superBlock->superBlockSize / blockSize) << BLOCK_MAP_SUPER_BLOCK_SIZE_SHIFT |
+ (superBlock->superBlockSize % blockSize == 0 ? 0 : BLOCK_MAP_FLAG_SUPER_BLOCK_SIZE_REMAINDER);
+
+ ioWriteVarIntU64(output, superBlockSizeEncoded);
+
+ if (superBlockSizeEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_SIZE_REMAINDER)
+ ioWriteVarIntU64(output, superBlock->superBlockSize % blockSize);
+
+ referenceData->superBlockSize = superBlock->superBlockSize;
+ }
}
sizeLast = (int64_t)superBlock->size;
- // Write all blocks in the current super block in packed format
- while (blockIdx < superBlockIdx)
+ // Write block total if the super block does not include all blocks with no block offset
+ if (superBlockEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_TOTAL_OFFSET)
{
- const BlockMapItem *const block = blockMapGet(this, blockIdx);
- ASSERT(block->block >= referenceData->block);
+ // Write total blocks in the super block
+ const uint64_t blockTotalEncoded =
+ (blockTotal - 1) << BLOCK_MAP_BLOCK_TOTAL_SHIFT |
+ (superBlock->block - referenceData->block > 0 ? BLOCK_MAP_FLAG_BLOCK_TOTAL_OFFSET : 0);
- // Block no only needs to be written when super block size does not equal block size
- if (!blockEqual)
+ ioWriteVarIntU64(output, blockTotalEncoded);
+
+ // If there is a gap in block no from the prior super block. This can happen when a super block is continued or
+ // had blocks at the beginning overridden by a newer super block.
+ if (blockTotalEncoded & BLOCK_MAP_FLAG_BLOCK_TOTAL_OFFSET)
{
- // Write block no as the difference from the prior block
- ioWriteVarIntU64(
- output,
- (blockIdx == superBlockIdx - 1 ? BLOCK_MAP_FLAG_LAST : 0) |
- (block->block - referenceData->block) << BLOCK_MAP_BLOCK_SHIFT);
-
- referenceData->block = block->block;
+ ioWriteVarIntU64(output, superBlock->block - referenceData->block);
+ referenceData->block = superBlock->block;
}
+ }
- // Write checksum
- ioWrite(output, BUF(block->checksum, checksumSize));
+ ASSERT(superBlock->block >= referenceData->block);
- blockIdx++;
+ // Increment reference block by number of blocks written
+ referenceData->block += blockTotal;
+
+ // Write checksums
+ for (; blockIdx < superBlockIdx; blockIdx++)
+ {
+ ASSERT(
+ superBlock == blockMapGet(this, blockIdx) ||
+ blockMapGet(this, blockIdx)->block == blockMapGet(this, blockIdx - 1)->block + 1);
+
+ ioWrite(output, BUF(blockMapGet(this, blockIdx)->checksum, checksumSize));
}
}
}
diff --git a/src/command/backup/blockMap.h b/src/command/backup/blockMap.h
index 99c4568ca..445c339d5 100644
--- a/src/command/backup/blockMap.h
+++ b/src/command/backup/blockMap.h
@@ -20,9 +20,10 @@ typedef struct BlockMap BlockMap;
typedef struct BlockMapItem
{
unsigned int reference; // Reference to backup where the block is stored
+ uint64_t superBlockSize; // Super block size
uint64_t bundleId; // Bundle where the block is stored (0 if not bundled)
uint64_t offset; // Offset of super block into the bundle
- uint64_t size; // Size of the super block (including compression, etc.)
+ uint64_t size; // Stored super block size (with compression, etc.)
uint64_t block; // Block no inside of super block
unsigned char checksum[XX_HASH_SIZE_MAX]; // Checksum of the block
} BlockMapItem;
@@ -38,7 +39,7 @@ blockMapNew(void)
}
// New block map from IO
-FN_EXTERN BlockMap *blockMapNewRead(IoRead *map, size_t checksumSize);
+FN_EXTERN BlockMap *blockMapNewRead(IoRead *map, size_t blockSize, size_t checksumSize);
/***********************************************************************************************************************************
Functions
@@ -52,7 +53,7 @@ blockMapAdd(BlockMap *const this, const BlockMapItem *const item)
}
// Write map to IO
-FN_EXTERN void blockMapWrite(const BlockMap *this, IoWrite *output, bool blockEqual, size_t checksumSize);
+FN_EXTERN void blockMapWrite(const BlockMap *this, IoWrite *output, size_t blockSize, size_t checksumSize);
/***********************************************************************************************************************************
Getters/Setters
diff --git a/src/command/restore/blockDelta.c b/src/command/restore/blockDelta.c
index 68300fd2e..4e1d9e92c 100644
--- a/src/command/restore/blockDelta.c
+++ b/src/command/restore/blockDelta.c
@@ -15,7 +15,8 @@ Object type
***********************************************************************************************************************************/
typedef struct BlockDeltaSuperBlock
{
- uint64_t size; // Size of super block
+ uint64_t superBlockSize; // Super block size
+ uint64_t size; // Stored size of superblock (with compression, etc.)
List *blockList; // Block list
} BlockDeltaSuperBlock;
@@ -37,10 +38,11 @@ struct BlockDelta
const BlockDeltaSuperBlock *superBlockData; // Current super block data
unsigned int superBlockIdx; // Current super block index
- unsigned int blockNo; // Block number in current super block
IoRead *chunkedRead; // Chunked read for current super block
const BlockDeltaBlock *blockData; // Current block data
unsigned int blockIdx; // Current block index
+ unsigned int blockTotal; // Block total for super block
+ unsigned int blockFindIdx; // Index of the block to find in the super block
BlockDeltaWrite write; // Block/offset to be returned for write
};
@@ -173,6 +175,7 @@ blockDeltaNew(
{
BlockDeltaSuperBlock blockDeltaSuperBlockNew =
{
+ .superBlockSize = blockMapItem->superBlockSize,
.size = blockMapItem->size,
.blockList = lstNewP(sizeof(BlockDeltaBlock)),
};
@@ -253,84 +256,65 @@ blockDeltaNext(BlockDelta *const this, const BlockDeltaRead *const readDelta, Io
// Set block info
this->blockIdx = 0;
- this->blockNo = 0;
- this->blockData = lstGet(this->superBlockData->blockList, this->blockIdx);
+ this->blockFindIdx = 0;
+ this->blockTotal =
+ (unsigned int)(this->superBlockData->superBlockSize / this->blockSize) +
+ (this->superBlockData->superBlockSize % this->blockSize == 0 ? 0 : 1);
+ this->blockData = lstGet(this->superBlockData->blockList, this->blockFindIdx);
}
- // Read encoded info about the block
- uint64_t blockEncoded = ioReadVarIntU64(this->chunkedRead);
-
- // Loop through blocks in the super block until a match with the map is found
- do
+ // Find required blocks in the super block
+ while (this->blockIdx < this->blockTotal)
{
- // Break out if encoded block info is zero and this is not the first block. This means the super block has ended when
- // all blocks are equal size.
- if (this->blockNo != 0 && blockEncoded == 0)
- {
- this->superBlockData = NULL;
- this->superBlockIdx++;
- break;
- }
+ // Read encoded info about the block, which is not used here
+ ioReadVarIntU64(this->chunkedRead);
// Apply block size limit if required and read the block
bufUsedSet(this->write.block, 0);
- if (blockEncoded & BLOCK_INCR_FLAG_SIZE)
- {
- size_t size = (size_t)ioReadVarIntU64(this->chunkedRead);
- bufLimitSet(this->write.block, size);
- }
+ if (this->blockIdx == this->blockTotal - 1 && this->superBlockData->superBlockSize % this->blockSize != 0)
+ bufLimitSet(this->write.block, (size_t)(this->superBlockData->superBlockSize % this->blockSize));
else
bufLimitClear(this->write.block);
ioRead(this->chunkedRead, this->write.block);
// If the block matches the block we are expecting
- if (this->blockNo == this->blockData->no)
+ if (this->blockIdx == this->blockData->no)
{
+ ASSERT(result == NULL);
+
this->write.offset = this->blockData->offset;
result = &this->write;
- this->blockIdx++;
+ this->blockFindIdx++;
// Get the next block if there are any more to read
- if (this->blockIdx < lstSize(this->superBlockData->blockList))
- {
- this->blockData = lstGet(this->superBlockData->blockList, this->blockIdx);
- }
- // Else stop processing if this is the last super block. This is only works for the last super block because
- // otherwise the blocks must be read sequentially.
- else if (this->superBlockIdx >= lstSize(readDelta->superBlockList) - 1)
- {
- this->superBlockData = NULL;
- this->superBlockIdx++;
- }
+ if (this->blockFindIdx < lstSize(this->superBlockData->blockList))
+ this->blockData = lstGet(this->superBlockData->blockList, this->blockFindIdx);
}
- // If the size was set (meaning this is the last block in the super block) it must also be the last block we are looking
- // for in the read. Partial blocks cannot happen in the middle of a read and there is code above to cut out early on the
- // last super block of a read when possible.
- ASSERT(blockEncoded ^ BLOCK_INCR_FLAG_SIZE || this->superBlockIdx >= lstSize(readDelta->superBlockList) - 1);
-
// Increment the block to read in the super block
- this->blockNo++;
+ this->blockIdx++;
// Break if there is a result
if (result != NULL)
break;
-
- // Read encoded info about the block
- blockEncoded = ioReadVarIntU64(this->chunkedRead);
}
- while (true);
// Break if there is a result
if (result != NULL)
break;
+
+ this->superBlockData = NULL;
+ this->superBlockIdx++;
}
// If no result then the super blocks have been read. Reset for the next read.
if (result == NULL)
{
+ ASSERT(this->superBlockIdx == lstSize(readDelta->superBlockList));
+ ASSERT(this->blockIdx == this->blockTotal);
+
this->superBlockData = NULL;
this->superBlockIdx = 0;
}
diff --git a/src/command/restore/file.c b/src/command/restore/file.c
index cf571a036..712526b06 100644
--- a/src/command/restore/file.c
+++ b/src/command/restore/file.c
@@ -295,7 +295,8 @@ restoreFile(
// Read block map. This will be compared to the block checksum list already created to determine which
// blocks need to be fetched from the repository. If we got here there must be at least one block to fetch.
- const BlockMap *const blockMap = blockMapNewRead(storageReadIo(repoFileRead), file->blockIncrChecksumSize);
+ const BlockMap *const blockMap = blockMapNewRead(
+ storageReadIo(repoFileRead), file->blockIncrSize, file->blockIncrChecksumSize);
// The repo file needs to be closed so that block lists can be read from the remote protocol
ioReadClose(storageReadIo(repoFileRead));
@@ -342,6 +343,8 @@ restoreFile(
deltaWrite = blockDeltaNext(blockDelta, read, storageReadIo(superBlockRead));
}
+
+ storageReadFree(superBlockRead);
}
// Close the file to complete the update
diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c
index ca490d561..e1540e92e 100644
--- a/test/src/module/command/backupTest.c
+++ b/test/src/module/command/backupTest.c
@@ -50,7 +50,8 @@ testBlockDelta(const BlockMap *const blockMap, const size_t blockSize, const siz
{
const BlockDeltaSuperBlock *const superBlock = lstGet(read->superBlockList, superBlockIdx);
- strCatFmt(result, " super block {size: %" PRIu64 "}\n", superBlock->size);
+ strCatFmt(
+ result, " super block {max: %" PRIu64 ", size: %" PRIu64 "}\n", superBlock->superBlockSize, superBlock->size);
for (unsigned int blockIdx = 0; blockIdx < lstSize(superBlock->blockList); blockIdx++)
{
@@ -205,7 +206,8 @@ testBackupValidateList(
ioReadOpen(storageReadIo(read));
- const BlockMap *const blockMap = blockMapNewRead(storageReadIo(read), file.blockIncrChecksumSize);
+ const BlockMap *const blockMap = blockMapNewRead(
+ storageReadIo(read), file.blockIncrSize, file.blockIncrChecksumSize);
// Build map log
String *const mapLog = strNew();
@@ -682,6 +684,7 @@ testRun(void)
BlockMapItem blockMapItem =
{
.reference = 128,
+ .superBlockSize = 1,
.bundleId = 0,
.offset = 0,
.size = 3,
@@ -694,6 +697,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 128,
+ .superBlockSize = 1,
.bundleId = 0,
.offset = 3,
.size = 5,
@@ -705,6 +709,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 0,
+ .superBlockSize = 1,
.bundleId = 1,
.offset = 1,
.size = 5,
@@ -716,6 +721,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 128,
+ .superBlockSize = 1,
.bundleId = 0,
.offset = 8,
.size = 99,
@@ -727,6 +733,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 0,
+ .superBlockSize = 1,
.bundleId = 1,
.offset = 7,
.size = 99,
@@ -739,6 +746,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 4,
+ .superBlockSize = 1,
.bundleId = 0,
.offset = 0,
.size = 8,
@@ -752,36 +760,36 @@ testRun(void)
Buffer *buffer = bufNew(256);
IoWrite *write = ioBufferWriteNewOpen(buffer);
- TEST_RESULT_VOID(blockMapWrite(blockMap, write, true, 5), "save");
+ TEST_RESULT_VOID(blockMapWrite(blockMap, write, 1, 5), "save");
ioWriteClose(write);
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, buffer),
- "02" // Blocks are equal
+ "00" // Version 0
"8008" // reference 128
- "06" // super block size 3
+ "18" // size 3
"eeee01ffff" // checksum
- "09" // super block size 5
+ "21" // size
"eeee02ffff" // checksum
"06" // reference 0
"01" // bundle 1
"01" // offset 1
- "01" // super block size 5
+ "01" // size 1
"eeee03ffff" // checksum
"8008" // reference 128
- "f902" // super block size 99
+ "e10b" // size 99
"eeee04ffff" // checksum
"04" // reference 0
"01" // offset 7
- "01" // super block size 99
+ "01" // size 99
"eeee05ffff" // checksum
"21" // reference 0
- "eb02" // super block size 8
+ "a90b" // size 8
"eeee88ffff", // checksum
"compare");
@@ -790,7 +798,7 @@ testRun(void)
Buffer *bufferCompare = bufNew(256);
write = ioBufferWriteNewOpen(bufferCompare);
- TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer), 5), write, true, 5), "read and save");
+ TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer), 1, 5), write, 1, 5), "read and save");
ioWriteClose(write);
TEST_RESULT_STR(strNewEncode(encodingHex, bufferCompare), strNewEncode(encodingHex, buffer), "compare");
@@ -799,23 +807,23 @@ testRun(void)
TEST_TITLE("equal block delta");
TEST_RESULT_STR_Z(
- testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer), 5), 8, 5),
+ testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer), 1, 5), 1, 5),
"read {reference: 128, bundleId: 0, offset: 0, size: 107}\n"
- " super block {size: 3}\n"
+ " super block {max: 1, size: 3}\n"
" block {no: 0, offset: 0}\n"
- " super block {size: 5}\n"
- " block {no: 0, offset: 8}\n"
- " super block {size: 99}\n"
- " block {no: 0, offset: 24}\n"
+ " super block {max: 1, size: 5}\n"
+ " block {no: 0, offset: 1}\n"
+ " super block {max: 1, size: 99}\n"
+ " block {no: 0, offset: 3}\n"
"read {reference: 4, bundleId: 0, offset: 0, size: 8}\n"
- " super block {size: 8}\n"
- " block {no: 0, offset: 40}\n"
+ " super block {max: 1, size: 8}\n"
+ " block {no: 0, offset: 5}\n"
"read {reference: 0, bundleId: 1, offset: 1, size: 5}\n"
- " super block {size: 5}\n"
- " block {no: 0, offset: 16}\n"
+ " super block {max: 1, size: 5}\n"
+ " block {no: 0, offset: 2}\n"
"read {reference: 0, bundleId: 1, offset: 7, size: 99}\n"
- " super block {size: 99}\n"
- " block {no: 0, offset: 32}\n",
+ " super block {max: 1, size: 99}\n"
+ " block {no: 0, offset: 4}\n",
"check delta");
// -------------------------------------------------------------------------------------------------------------------------
@@ -826,6 +834,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 0,
+ .superBlockSize = 6,
.offset = 0,
.size = 4,
.block = 0,
@@ -837,6 +846,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 0,
+ .superBlockSize = 6,
.offset = 0,
.size = 4,
.block = 1,
@@ -848,6 +858,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 0,
+ .superBlockSize = 6,
.offset = 4,
.size = 5,
.block = 0,
@@ -859,6 +870,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 1,
+ .superBlockSize = 3,
.offset = 0,
.size = 99,
.block = 0,
@@ -870,6 +882,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 0,
+ .superBlockSize = 6,
.offset = 4,
.size = 5,
.block = 3,
@@ -881,6 +894,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 2,
+ .superBlockSize = 2,
.offset = 0,
.size = 1,
.block = 0,
@@ -892,6 +906,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 0,
+ .superBlockSize = 6,
.offset = 4,
.size = 5,
.block = 5,
@@ -903,6 +918,7 @@ testRun(void)
blockMapItem = (BlockMapItem)
{
.reference = 0,
+ .superBlockSize = 2,
.offset = 9,
.size = 6,
.block = 0,
@@ -911,12 +927,24 @@ testRun(void)
TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
+ blockMapItem = (BlockMapItem)
+ {
+ .reference = 1,
+ .superBlockSize = 3,
+ .offset = 99,
+ .size = 1,
+ .block = 1,
+ .checksum = {0xee, 0xee, 0x09, 0, 0, 0, 0xff, 0xff},
+ };
+
+ TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
+
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write unequal block map");
buffer = bufNew(256);
write = ioBufferWriteNewOpen(buffer);
- TEST_RESULT_VOID(blockMapWrite(blockMap, write, false, 8), "save");
+ TEST_RESULT_VOID(blockMapWrite(blockMap, write, 3, 8), "save");
ioWriteClose(write);
TEST_RESULT_STR_Z(
@@ -924,37 +952,43 @@ testRun(void)
"00" // Blocks are unequal
"00" // reference 0
- "08" // size 4
- "00" // block 0
+ "22" // size 4
+ "04" // super block size 6
"eeee01000000ffff" // checksum
-
- "03" // block 1
"eeee02000000ffff" // checksum
- "05" // size 5
- "01" // block 0
+ "15" // size 5
+ "00" // block total 1
"eeee03000000ffff" // checksum
"08" // reference 1
- "f902" // size 99
- "01" // block 0
+ "e10b" // size 99
"eeee04000000ffff" // checksum
"06" // reference 0
- "07" // block 3
+ "01" // block total 1
+ "02" // block 3
"eeee05000000ffff" // checksum
"10" // reference 2
- "0f" // size 1
- "01" // block 0
+ "3b01" // size 1
+ "02" // super block size 2
"eeee06000000ffff" // checksum
- "03" // reference 0
- "05" // size 5
+ "02" // reference 0
+ "01" // block total 1
+ "01" // block 5
"eeee07000000ffff" // checksum
- "05" // size 6
- "01" // block
- "eeee08000000ffff", // checksum
+
+ "13" // size 6
+ "0102" // size 2
+ "eeee08000000ffff" // checksum
+
+ "09" // reference 1
+ "4d" // size 1
+ "01" // block total 1
+ "01" // block 1
+ "eeee09000000ffff", // checksum
"compare");
// -------------------------------------------------------------------------------------------------------------------------
@@ -962,7 +996,7 @@ testRun(void)
bufferCompare = bufNew(256);
write = ioBufferWriteNewOpen(bufferCompare);
- TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer), 8), write, false, 8), "read and save");
+ TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer), 3, 8), write, 3, 8), "read and save");
ioWriteClose(write);
TEST_RESULT_STR(strNewEncode(encodingHex, bufferCompare), strNewEncode(encodingHex, buffer), "compare");
@@ -971,23 +1005,25 @@ testRun(void)
TEST_TITLE("unequal block delta");
TEST_RESULT_STR_Z(
- testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer), 8), 8, 8),
+ testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer), 3, 8), 3, 8),
"read {reference: 2, bundleId: 0, offset: 0, size: 1}\n"
- " super block {size: 1}\n"
- " block {no: 0, offset: 40}\n"
- "read {reference: 1, bundleId: 0, offset: 0, size: 99}\n"
- " super block {size: 99}\n"
- " block {no: 0, offset: 24}\n"
+ " super block {max: 2, size: 1}\n"
+ " block {no: 0, offset: 15}\n"
+ "read {reference: 1, bundleId: 0, offset: 0, size: 100}\n"
+ " super block {max: 3, size: 99}\n"
+ " block {no: 0, offset: 9}\n"
+ " super block {max: 3, size: 1}\n"
+ " block {no: 1, offset: 24}\n"
"read {reference: 0, bundleId: 0, offset: 0, size: 15}\n"
- " super block {size: 4}\n"
+ " super block {max: 6, size: 4}\n"
" block {no: 0, offset: 0}\n"
- " block {no: 1, offset: 8}\n"
- " super block {size: 5}\n"
- " block {no: 0, offset: 16}\n"
- " block {no: 3, offset: 32}\n"
- " block {no: 5, offset: 48}\n"
- " super block {size: 6}\n"
- " block {no: 0, offset: 56}\n",
+ " block {no: 1, offset: 3}\n"
+ " super block {max: 6, size: 5}\n"
+ " block {no: 0, offset: 6}\n"
+ " block {no: 3, offset: 12}\n"
+ " block {no: 5, offset: 18}\n"
+ " super block {max: 2, size: 6}\n"
+ " block {no: 0, offset: 21}\n",
"check delta");
}
@@ -1042,19 +1078,19 @@ testRun(void)
uint64_t mapSize;
TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size");
- TEST_RESULT_UINT(mapSize, 11, "map size");
+ TEST_RESULT_UINT(mapSize, 13, "map size");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
- "02010201313200", // block 0
+ "020031023200", // block 0
"block list");
const Buffer *map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
TEST_RESULT_STR_Z(
- testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
- "read {reference: 0, bundleId: 0, offset: 0, size: 7}\n"
- " super block {size: 7}\n"
+ testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 3, 8), 3, 8),
+ "read {reference: 0, bundleId: 0, offset: 0, size: 6}\n"
+ " super block {max: 2, size: 6}\n"
" block {no: 0, offset: 0}\n",
"check delta");
@@ -1068,7 +1104,7 @@ testRun(void)
TEST_RESULT_VOID(
ioFilterGroupAdd(
ioWriteFilterGroup(write),
- blockIncrNewPack(ioFilterParamList(blockIncrNew(5, 3, 8, 2, 4, 5, NULL, NULL, NULL)))),
+ blockIncrNewPack(ioFilterParamList(blockIncrNew(2, 3, 8, 2, 4, 5, NULL, NULL, NULL)))),
"block incr");
TEST_RESULT_VOID(ioWriteOpen(write), "open");
TEST_RESULT_VOID(ioWrite(write, source), "write");
@@ -1079,21 +1115,21 @@ testRun(void)
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
- "020041014243020000" // block 0
- "02025801595a020000" // block 1
- "020231013233020000", // block 2
+ "02004101424300" // block 0
+ "02015801595a00" // block 1
+ "02013101323300", // block 2
"block list");
map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
TEST_RESULT_STR_Z(
- testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
- "read {reference: 2, bundleId: 4, offset: 5, size: 27}\n"
- " super block {size: 9}\n"
+ testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 3, 8), 3, 8),
+ "read {reference: 2, bundleId: 4, offset: 5, size: 21}\n"
+ " super block {max: 3, size: 7}\n"
" block {no: 0, offset: 0}\n"
- " super block {size: 9}\n"
+ " super block {max: 3, size: 7}\n"
" block {no: 0, offset: 3}\n"
- " super block {size: 9}\n"
+ " super block {max: 3, size: 7}\n"
" block {no: 0, offset: 6}\n",
"check delta");
@@ -1115,27 +1151,27 @@ testRun(void)
TEST_RESULT_VOID(ioWriteClose(write), "close");
TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size");
- TEST_RESULT_UINT(mapSize, 42, "map size");
+ TEST_RESULT_UINT(mapSize, 44, "map size");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
- "0300414302430000" // block 0
- "0307014000", // block 3
+ "03004143044300" // block 0
+ "02034000", // block 3
"block list");
map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
TEST_RESULT_STR_Z(
- testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
- "read {reference: 3, bundleId: 0, offset: 0, size: 13}\n"
- " super block {size: 8}\n"
+ testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 3, 8), 3, 8),
+ "read {reference: 3, bundleId: 0, offset: 0, size: 11}\n"
+ " super block {max: 3, size: 7}\n"
" block {no: 0, offset: 0}\n"
- " super block {size: 5}\n"
+ " super block {max: 1, size: 4}\n"
" block {no: 0, offset: 9}\n"
- "read {reference: 2, bundleId: 4, offset: 14, size: 18}\n"
- " super block {size: 9}\n"
+ "read {reference: 2, bundleId: 4, offset: 12, size: 14}\n"
+ " super block {max: 3, size: 7}\n"
" block {no: 0, offset: 3}\n"
- " super block {size: 9}\n"
+ " super block {max: 3, size: 7}\n"
" block {no: 0, offset: 6}\n",
"check delta");
@@ -1157,24 +1193,24 @@ testRun(void)
TEST_RESULT_VOID(ioWriteClose(write), "close");
TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size");
- TEST_RESULT_UINT(mapSize, 33, "map size");
+ TEST_RESULT_UINT(mapSize, 32, "map size");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
"020041014243" // super block 0 / block 0
- "01025801595a020000" // super block 0 / block 1
- "020231013233020000", // block 2
+ "01015801595a00" // super block 0 / block 1
+ "02013101323300", // block 2
"block list");
map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
TEST_RESULT_STR_Z(
- testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
- "read {reference: 2, bundleId: 4, offset: 5, size: 24}\n"
- " super block {size: 15}\n"
+ testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 3, 8), 3, 8),
+ "read {reference: 2, bundleId: 4, offset: 5, size: 20}\n"
+ " super block {max: 6, size: 13}\n"
" block {no: 0, offset: 0}\n"
" block {no: 1, offset: 3}\n"
- " super block {size: 9}\n"
+ " super block {max: 3, size: 7}\n"
" block {no: 0, offset: 6}\n",
"check delta");
@@ -3653,7 +3689,7 @@ testRun(void)
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/0, 16.0KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16383, 8KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24575, 16KB, [PCT]) checksum [SHA1]\n"
- "P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (bundle 1/40998, 2B, [PCT]) checksum [SHA1]\n"
+ "P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (bundle 1/40996, 2B, [PCT]) checksum [SHA1]\n"
"P00 INFO: execute non-exclusive backup stop and wait for all WAL segments to archive\n"
"P00 INFO: backup stop archive = 0000000105DBF06000000001, lsn = 5dbf060/300000\n"
"P00 DETAIL: wrote 'backup_label' file returned from backup stop function\n"
@@ -3683,9 +3719,9 @@ testRun(void)
",\"timestamp\":1572800000}\n"
"pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
",\"timestamp\":1572800002}\n"
- "pg_data/block-incr-grow={\"bi\":1,\"bim\":26,\"checksum\":\"ebdd38b69cd5b9f2d00d273c981e16960fbbb4f7\""
+ "pg_data/block-incr-grow={\"bi\":1,\"bim\":24,\"checksum\":\"ebdd38b69cd5b9f2d00d273c981e16960fbbb4f7\""
",\"size\":24576,\"timestamp\":1572800000}\n"
- "pg_data/block-incr-shrink={\"bi\":1,\"bim\":30,\"checksum\":\"ce5f8864058b1bb274244b512cb9641355987134\""
+ "pg_data/block-incr-shrink={\"bi\":1,\"bim\":29,\"checksum\":\"ce5f8864058b1bb274244b512cb9641355987134\""
",\"size\":16385,\"timestamp\":1572800000}\n"
"pg_data/global/pg_control={\"size\":8192,\"timestamp\":1572800000}\n"
"pg_data/grow-to-block-incr={\"checksum\":\"f5a5c308cf5fcb52bccebe2365f8ed56acbcc41d\",\"size\":16383"
@@ -3764,8 +3800,8 @@ testRun(void)
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-larger (1.4MB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-grow (128KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/0, 16KB, [PCT]) checksum [SHA1]\n"
- "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16420, 8KB, [PCT]) checksum [SHA1]\n"
- "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24612, 16.0KB, [PCT]) checksum [SHA1]\n"
+ "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16418, 8KB, [PCT]) checksum [SHA1]\n"
+ "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24610, 16.0KB, [PCT]) checksum [SHA1]\n"
"P00 DETAIL: reference pg_data/PG_VERSION to 20191103-165320F\n"
"P00 INFO: execute non-exclusive backup stop and wait for all WAL segments to archive\n"
"P00 INFO: backup stop archive = 0000000105DC213000000001, lsn = 5dc2130/300000\n"
@@ -3796,14 +3832,14 @@ testRun(void)
",\"size\":2,\"timestamp\":1572800000}\n"
"pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
",\"timestamp\":1573000002}\n"
- "pg_data/block-incr-grow={\"bi\":1,\"bim\":123,\"checksum\":\"1ddde8db92dd9019be0819ae4f9ad9cea2fae399\""
+ "pg_data/block-incr-grow={\"bi\":1,\"bim\":114,\"checksum\":\"1ddde8db92dd9019be0819ae4f9ad9cea2fae399\""
",\"size\":131072,\"timestamp\":1573000000}\n"
- "pg_data/block-incr-larger={\"bi\":8,\"bic\":7,\"bim\":194"
+ "pg_data/block-incr-larger={\"bi\":8,\"bic\":7,\"bim\":173"
",\"checksum\":\"eec53a6da79c00b3c658a7e09f44b3e9efefd960\",\"size\":1507328,\"timestamp\":1573000000}\n"
"pg_data/block-incr-shrink={\"checksum\":\"1c6a17f67562d8b3f64f1b5f2ee592a4c2809b3b\",\"size\":16383"
",\"timestamp\":1573000000}\n"
"pg_data/global/pg_control={\"size\":8192,\"timestamp\":1573000000}\n"
- "pg_data/grow-to-block-incr={\"bi\":1,\"bim\":27,\"checksum\":\"4f560611d9dc9212432970e5c4bec15d876c226e\","
+ "pg_data/grow-to-block-incr={\"bi\":1,\"bim\":26,\"checksum\":\"4f560611d9dc9212432970e5c4bec15d876c226e\","
"\"size\":16385,\"timestamp\":1573000000}\n"
"pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19"
",\"timestamp\":1573000002}\n"