mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2024-12-12 10:04:14 +02:00
Block incremental map fixes and improvements.
Bug Fixes: * Remove the distinction between maps where super block size is equal to block size and maps where they are not. In practice, maps with equal blocks are now rare and most of the optimizations can be applied directly to super blocks where the blocks are equal. This fixes a bug where a map that was created with equal size blocks and then converted to differing block sizes would generate an invalid map. * Free reads during restore to avoid running out of file handles. Improvements: * Store super block sizes in the block map. This allows the final block size to be removed from the block list and provides a more optimal restore and better potential for analysis. * Always round the super block size up to the next block size. This makes the number of blocks per super block more predictable. * Allow super block sizes to be changed at will in the map. The first case for this is to store the reduced super block size required when the last super block is short but it could be used to dynamically change the super block size to optimize compression. * Store a block count rather than a list of blocks in a super block. Blocks must always be sequential, though there may be an offset to the first block in a super block. This saves 11-14% on space for checksum sizes 6-7. * In the case that all the blocks for a super block are present, and there is no offset, the block size is omitted.
This commit is contained in:
David Steele
GitHub
parent
5c1f78d4dd
commit
c30d3e439b
@ -54,6 +54,9 @@
|
||||
<commit subject="Update default block size and super block values based on testing."/>
|
||||
<commit subject="Ensure no continuations when block size equals super block size."/>
|
||||
<commit subject="Fix typo in blockIncrProcess()."/>
|
||||
<commit subject="Block incremental map fixes and improvements.">
|
||||
<github-pull-request id="2026"/>
|
||||
</commit>
|
||||
|
||||
<release-item-contributor-list>
|
||||
<release-item-contributor id="david.steele"/>
|
||||
|
||||
@ -271,20 +271,19 @@ backupInit(const InfoBackup *infoBackup)
|
||||
Build block incremental maps
|
||||
***********************************************************************************************************************************/
|
||||
// Size map. Block size is increased when the block map would be larger than a single block. The break can be calculated with this
|
||||
// formula: [block size in KiB] / (1024 / [block size in KiB] * [checksum size + 1]) * 1073741824.
|
||||
// formula: [block size in KiB] / (1024 / [block size in KiB] * [checksum size]) * 1073741824.
|
||||
static const ManifestBlockIncrSizeMap manifestBlockIncrSizeMapDefault[] =
|
||||
{
|
||||
{.fileSize = 968 * 1024 * 1024, .blockSize = 96 * 1024},
|
||||
{.fileSize = 800 * 1024 * 1024, .blockSize = 88 * 1024},
|
||||
{.fileSize = 648 * 1024 * 1024, .blockSize = 80 * 1024},
|
||||
{.fileSize = 512 * 1024 * 1024, .blockSize = 72 * 1024},
|
||||
{.fileSize = 392 * 1024 * 1024, .blockSize = 64 * 1024},
|
||||
{.fileSize = 288 * 1024 * 1024, .blockSize = 56 * 1024},
|
||||
{.fileSize = 200 * 1024 * 1024, .blockSize = 48 * 1024},
|
||||
{.fileSize = 128 * 1024 * 1024, .blockSize = 40 * 1024},
|
||||
{.fileSize = 84256 * 1024, .blockSize = 32 * 1024}, // These do not come out evenly because the map record size is 7
|
||||
{.fileSize = 37448 * 1024, .blockSize = 24 * 1024}, // instead of 8 in the rows above
|
||||
{.fileSize = 9360 * 1024, .blockSize = 16 * 1024},
|
||||
{.fileSize = 914 * 1024 * 1024, .blockSize = 88 * 1024},
|
||||
{.fileSize = 740 * 1024 * 1024, .blockSize = 80 * 1024},
|
||||
{.fileSize = 585 * 1024 * 1024, .blockSize = 72 * 1024},
|
||||
{.fileSize = 448 * 1024 * 1024, .blockSize = 64 * 1024},
|
||||
{.fileSize = 329 * 1024 * 1024, .blockSize = 56 * 1024},
|
||||
{.fileSize = 228 * 1024 * 1024, .blockSize = 48 * 1024},
|
||||
{.fileSize = 146 * 1024 * 1024, .blockSize = 40 * 1024},
|
||||
{.fileSize = 96 * 1024 * 1024, .blockSize = 32 * 1024},
|
||||
{.fileSize = 43 * 1024 * 1024, .blockSize = 24 * 1024},
|
||||
{.fileSize = 11 * 1024 * 1024, .blockSize = 16 * 1024},
|
||||
{.fileSize = 16 * 1024, .blockSize = 8 * 1024},
|
||||
};
|
||||
|
||||
|
||||
@ -174,15 +174,8 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output)
|
||||
ioWriteOpen(this->blockOutWrite);
|
||||
}
|
||||
|
||||
// Write the block no as a delta of the prior block no. If the size of the last block is smaller than block
|
||||
// size then write the smaller block size.
|
||||
const uint64_t blockEncoded = bufUsed(this->block) < this->blockSize ? BLOCK_INCR_FLAG_SIZE : 0;
|
||||
|
||||
ioWriteVarIntU64(
|
||||
this->blockOutWrite, blockEncoded | ((this->blockNo - this->blockNoLast) << BLOCK_INCR_BLOCK_SHIFT));
|
||||
|
||||
if (blockEncoded & BLOCK_INCR_FLAG_SIZE)
|
||||
ioWriteVarIntU64(this->blockOutWrite, bufUsed(this->block));
|
||||
// Write the block no as a delta of the prior block no
|
||||
ioWriteVarIntU64(this->blockOutWrite, this->blockNo - this->blockNoLast);
|
||||
|
||||
// Copy block data through the filters
|
||||
ioCopyP(ioBufferReadNewOpen(this->block), this->blockOutWrite);
|
||||
@ -193,6 +186,7 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output)
|
||||
BlockMapItem blockMapItem =
|
||||
{
|
||||
.reference = this->reference,
|
||||
.superBlockSize = this->superBlockSize,
|
||||
.bundleId = this->bundleId,
|
||||
.offset = this->blockOffset,
|
||||
.block = this->superBlockNo,
|
||||
@ -225,22 +219,20 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output)
|
||||
// Write the super block
|
||||
if (this->blockOutWrite != NULL && (this->done || this->blockOutSize >= this->superBlockSize))
|
||||
{
|
||||
// Explicitly terminate the block if all block sizes are equal. This is not required if the last block is smaller than
|
||||
// the block size.
|
||||
if (this->blockOutSize % this->blockSize == 0)
|
||||
ioWriteVarIntU64(this->blockOutWrite, 0);
|
||||
|
||||
// Close write
|
||||
ioWriteClose(this->blockOutWrite);
|
||||
|
||||
// Update size for items already added to the block map
|
||||
// Update size and super block size for items already added to the block map
|
||||
const uint64_t blockOutSize = pckReadU64P(
|
||||
ioFilterGroupResultP(ioWriteFilterGroup(this->blockOutWrite), SIZE_FILTER_TYPE));
|
||||
|
||||
for (unsigned int blockMapIdx = 0; blockMapIdx < lstSize(this->blockOutList); blockMapIdx++)
|
||||
{
|
||||
blockMapGet(
|
||||
this->blockMapOut, *(unsigned int *)lstGet(this->blockOutList, blockMapIdx))->size = blockOutSize;
|
||||
BlockMapItem *const blockMapItem = blockMapGet(
|
||||
this->blockMapOut, *(unsigned int *)lstGet(this->blockOutList, blockMapIdx));
|
||||
|
||||
blockMapItem->size = blockOutSize;
|
||||
blockMapItem->superBlockSize = this->blockOutSize;
|
||||
}
|
||||
|
||||
lstFree(this->blockOutList);
|
||||
@ -273,7 +265,7 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output)
|
||||
|
||||
// Write the map
|
||||
ioWriteOpen(write);
|
||||
blockMapWrite(this->blockMapOut, write, this->blockSize == this->superBlockSize, this->checksumSize);
|
||||
blockMapWrite(this->blockMapOut, write, this->blockSize, this->checksumSize);
|
||||
ioWriteClose(write);
|
||||
|
||||
// Get total bytes written for the map
|
||||
@ -407,7 +399,7 @@ blockIncrNew(
|
||||
*driver = (BlockIncr)
|
||||
{
|
||||
.memContext = memContextCurrent(),
|
||||
.superBlockSize = superBlockSize,
|
||||
.superBlockSize = (superBlockSize / blockSize + (superBlockSize % blockSize == 0 ? 0 : 1)) * blockSize,
|
||||
.blockSize = blockSize,
|
||||
.checksumSize = checksumSize,
|
||||
.reference = reference,
|
||||
@ -418,10 +410,6 @@ blockIncrNew(
|
||||
.blockMapOut = blockMapNew(),
|
||||
};
|
||||
|
||||
// If there will be less than two blocks per super block then set them equal to save space in the map
|
||||
if (superBlockSize < blockSize * 2)
|
||||
driver->superBlockSize = blockSize;
|
||||
|
||||
// Duplicate compress filter
|
||||
if (compress != NULL)
|
||||
{
|
||||
@ -442,7 +430,7 @@ blockIncrNew(
|
||||
|
||||
MEM_CONTEXT_PRIOR_BEGIN()
|
||||
{
|
||||
driver->blockMapPrior = blockMapNewRead(read, checksumSize);
|
||||
driver->blockMapPrior = blockMapNewRead(read, blockSize, checksumSize);
|
||||
}
|
||||
MEM_CONTEXT_PRIOR_END();
|
||||
}
|
||||
|
||||
@ -20,9 +20,6 @@ consists of the following, compressed and encrypted as required:
|
||||
when the block list is being read sequentially, e.g. during verification. If blocks are accessed from the map then the block
|
||||
number is already known and the delta can be ignored.
|
||||
|
||||
- If the block number above has BLOCK_INCR_FLAG_SIZE set then the block size is different from the default and must be read. This is
|
||||
because it is at the end of the file and so also indicates the end of the super block.
|
||||
|
||||
- Block data.
|
||||
|
||||
The super block list is followed by the block map, which is encrypted separately when required but not compressed. The return value
|
||||
@ -48,12 +45,6 @@ Filter type constant
|
||||
***********************************************************************************************************************************/
|
||||
#define BLOCK_INCR_FILTER_TYPE STRID5("blk-incr", 0x90dc9dad820)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constants needed to read the block number in a super block
|
||||
***********************************************************************************************************************************/
|
||||
#define BLOCK_INCR_FLAG_SIZE 1 // Does the block have a different size than the default?
|
||||
#define BLOCK_INCR_BLOCK_SHIFT 1 // Shift required to get block number
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
***********************************************************************************************************************************/
|
||||
|
||||
@ -40,22 +40,27 @@ References, super blocks, and blocks are encoded with a bit that indicates when
|
||||
#define BLOCK_MAP_FLAG_CONTINUE 2 // Reference continues a prior super block
|
||||
#define BLOCK_MAP_FLAG_CONTINUE_LAST 4 // Continued super block is last for the reference
|
||||
#define BLOCK_MAP_REFERENCE_SHIFT 3 // Shift bits for reference
|
||||
#define BLOCK_MAP_SUPER_BLOCK_SHIFT 1 // Shift bits for super block
|
||||
#define BLOCK_MAP_BLOCK_SHIFT 1 // Shift bits for block
|
||||
#define BLOCK_MAP_FLAG_SUPER_BLOCK_SIZE_REMAINDER 1 // Remainder for super block size
|
||||
#define BLOCK_MAP_SUPER_BLOCK_SIZE_SHIFT 1 // Shift bits for super block size
|
||||
#define BLOCK_MAP_FLAG_SUPER_BLOCK_CHANGE 2 // The super block size has changed
|
||||
#define BLOCK_MAP_FLAG_SUPER_BLOCK_TOTAL_OFFSET 4 // Block total/offset when not complete
|
||||
#define BLOCK_MAP_SUPER_BLOCK_SHIFT 3 // Shift bits for super block
|
||||
#define BLOCK_MAP_FLAG_BLOCK_TOTAL_OFFSET 1 // Block total has an offset
|
||||
#define BLOCK_MAP_BLOCK_TOTAL_SHIFT 1 // Shift bits for block total
|
||||
|
||||
typedef enum
|
||||
{
|
||||
blockMapFlagVersion = 0, // Version (currently always 0)
|
||||
blockMapFlagEqual = 1, // Are blocks and super blocks equal size?
|
||||
} BlockMapFlag;
|
||||
|
||||
// Stores current information about a reference to avoid needed to encode it again
|
||||
typedef struct BlockMapReference
|
||||
{
|
||||
unsigned int reference; // Reference
|
||||
uint64_t superBlockSize; // Super block size
|
||||
uint64_t bundleId; // Bundle id
|
||||
uint64_t offset; // Offset
|
||||
uint64_t size; // Super block size
|
||||
uint64_t size; // Stored super block size (with compression, etc.)
|
||||
uint64_t block; // Block no
|
||||
} BlockMapReference;
|
||||
|
||||
@ -80,7 +85,7 @@ lstComparatorBlockMapReference(const void *const blockMapRef1, const void *const
|
||||
}
|
||||
|
||||
FN_EXTERN BlockMap *
|
||||
blockMapNewRead(IoRead *const map, size_t checksumSize)
|
||||
blockMapNewRead(IoRead *const map, const size_t blockSize, const size_t checksumSize)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||
FUNCTION_LOG_PARAM(IO_READ, map);
|
||||
@ -88,11 +93,7 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
|
||||
|
||||
// Read flags. Currently the version flag must always be zero. This may be used in the future to indicate if the map version
|
||||
// has changed.
|
||||
const uint64_t flags = ioReadVarIntU64(map);
|
||||
CHECK(FormatError, (flags & (1 << blockMapFlagVersion)) == 0, "block map version must be zero");
|
||||
|
||||
// Are the super block and block size equal?
|
||||
const bool blockEqual = flags & (1 << blockMapFlagEqual);
|
||||
CHECK(FormatError, (ioReadVarIntU64(map) & (1 << blockMapFlagVersion)) == 0, "block map version must be zero");
|
||||
|
||||
// Read all references in packed format
|
||||
BlockMap *const this = blockMapNew();
|
||||
@ -119,10 +120,14 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
|
||||
if (referenceEncoded & BLOCK_MAP_FLAG_OFFSET)
|
||||
blockMapItem.offset = ioReadVarIntU64(map);
|
||||
|
||||
// Default super block size
|
||||
blockMapItem.superBlockSize = blockSize;
|
||||
|
||||
// Add reference to list
|
||||
BlockMapReference referenceDataAdd =
|
||||
{
|
||||
.reference = blockMapItem.reference,
|
||||
.superBlockSize = blockMapItem.superBlockSize,
|
||||
.bundleId = blockMapItem.bundleId,
|
||||
.offset = blockMapItem.offset,
|
||||
};
|
||||
@ -132,13 +137,12 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
|
||||
// Else this reference has been read before
|
||||
else
|
||||
{
|
||||
blockMapItem.superBlockSize = referenceData->superBlockSize;
|
||||
blockMapItem.bundleId = referenceData->bundleId;
|
||||
|
||||
// If the reference is continued use the prior offset and size values
|
||||
if (referenceEncoded & BLOCK_MAP_FLAG_CONTINUE)
|
||||
{
|
||||
ASSERT(!blockEqual);
|
||||
|
||||
blockMapItem.offset = referenceData->offset;
|
||||
blockMapItem.size = referenceData->size;
|
||||
referenceContinue = true;
|
||||
@ -168,6 +172,7 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
|
||||
if (referenceEncoded & BLOCK_MAP_FLAG_CONTINUE_LAST)
|
||||
superBlockEncoded = BLOCK_MAP_FLAG_LAST;
|
||||
|
||||
superBlockEncoded |= BLOCK_MAP_FLAG_SUPER_BLOCK_TOTAL_OFFSET;
|
||||
referenceContinue = false;
|
||||
}
|
||||
// Else read the super block size for the reference
|
||||
@ -175,14 +180,24 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
|
||||
{
|
||||
superBlockEncoded = ioReadVarIntU64(map);
|
||||
|
||||
// If this is the first size read then just read the size
|
||||
if (sizeLast == 0)
|
||||
{
|
||||
// If this is the first size read then just read the size. Otherwise read the difference from the prior size and
|
||||
// add sizeLast.
|
||||
blockMapItem.size = superBlockEncoded >> BLOCK_MAP_SUPER_BLOCK_SHIFT;
|
||||
|
||||
if (sizeLast != 0)
|
||||
blockMapItem.size = (uint64_t)(cvtInt64FromZigZag(blockMapItem.size) + sizeLast);
|
||||
|
||||
// If the super block size has changed then read it
|
||||
if (superBlockEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_CHANGE)
|
||||
{
|
||||
const uint64_t superBlockSizeEncoded = ioReadVarIntU64(map);
|
||||
blockMapItem.superBlockSize = (superBlockSizeEncoded >> BLOCK_MAP_SUPER_BLOCK_SIZE_SHIFT) * blockSize;
|
||||
|
||||
if (superBlockSizeEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_SIZE_REMAINDER)
|
||||
blockMapItem.superBlockSize += ioReadVarIntU64(map);
|
||||
|
||||
referenceData->superBlockSize = blockMapItem.superBlockSize;
|
||||
}
|
||||
// Else read the difference from the prior size and apply to sizeLast
|
||||
else
|
||||
blockMapItem.size = (uint64_t)(cvtInt64FromZigZag(superBlockEncoded >> BLOCK_MAP_SUPER_BLOCK_SHIFT) + sizeLast);
|
||||
|
||||
// Set offset, size, and block for the super block
|
||||
if (superBlockFirst)
|
||||
@ -198,19 +213,26 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
|
||||
sizeLast = (int64_t)blockMapItem.size;
|
||||
superBlockFirst = false;
|
||||
|
||||
// Read all blocks in the current super block in packed format
|
||||
do
|
||||
{
|
||||
// Block no only needs to be read when super block size does not equal block size
|
||||
uint64_t blockEncoded = BLOCK_MAP_FLAG_LAST;
|
||||
// Read or calculate block total
|
||||
uint64_t blockTotal;
|
||||
|
||||
if (!blockEqual)
|
||||
if (superBlockEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_TOTAL_OFFSET)
|
||||
{
|
||||
// Read block no as the difference from the prior block
|
||||
blockEncoded = ioReadVarIntU64(map);
|
||||
blockMapItem.block = (blockEncoded >> BLOCK_MAP_BLOCK_SHIFT) + referenceData->block;
|
||||
referenceData->block = blockMapItem.block;
|
||||
const uint64_t blockTotalEncoded = ioReadVarIntU64(map);
|
||||
blockTotal = (blockTotalEncoded >> BLOCK_MAP_BLOCK_TOTAL_SHIFT) + 1;
|
||||
|
||||
// Offset block no from expected
|
||||
if (blockTotalEncoded & BLOCK_MAP_FLAG_BLOCK_TOTAL_OFFSET)
|
||||
referenceData->block += ioReadVarIntU64(map);
|
||||
}
|
||||
else
|
||||
blockTotal = blockMapItem.superBlockSize / blockSize + (blockMapItem.superBlockSize % blockSize == 0 ? 0 : 1);
|
||||
|
||||
// Read checksums
|
||||
for (uint64_t blockIdx = 0; blockIdx < blockTotal; blockIdx++)
|
||||
{
|
||||
// Set block no
|
||||
blockMapItem.block = referenceData->block + blockIdx;
|
||||
|
||||
// Read checksum
|
||||
bufUsedZero(checksum);
|
||||
@ -219,12 +241,10 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
|
||||
|
||||
// Add to block list
|
||||
lstAdd((List *)this, &blockMapItem);
|
||||
|
||||
// Break when this is the last block in the super block
|
||||
if (blockEncoded & BLOCK_MAP_FLAG_LAST)
|
||||
break;
|
||||
}
|
||||
while (true);
|
||||
|
||||
// Update block in reference with all blocks read
|
||||
referenceData->block += blockTotal;
|
||||
|
||||
// Update offset with the super block size
|
||||
blockMapItem.offset += blockMapItem.size;
|
||||
@ -249,21 +269,22 @@ blockMapNewRead(IoRead *const map, size_t checksumSize)
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
FN_EXTERN void
|
||||
blockMapWrite(const BlockMap *const this, IoWrite *const output, const bool blockEqual, const size_t checksumSize)
|
||||
blockMapWrite(const BlockMap *const this, IoWrite *const output, const size_t blockSize, const size_t checksumSize)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||
FUNCTION_LOG_PARAM(BLOCK_MAP, this);
|
||||
FUNCTION_LOG_PARAM(IO_WRITE, output);
|
||||
FUNCTION_LOG_PARAM(BOOL, blockEqual);
|
||||
FUNCTION_LOG_PARAM(SIZE, blockSize);
|
||||
FUNCTION_LOG_PARAM(SIZE, checksumSize);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
ASSERT(this != NULL);
|
||||
ASSERT(blockMapSize(this) > 0);
|
||||
ASSERT(blockSize > 0);
|
||||
ASSERT(output != NULL);
|
||||
|
||||
// Set flag to indicate that super block and block size are equal
|
||||
ioWriteVarIntU64(output, blockEqual ? 1 << blockMapFlagEqual : 0);
|
||||
// Write flags
|
||||
ioWriteVarIntU64(output, 0);
|
||||
|
||||
// Write all references in packed format
|
||||
List *const refList = lstNewP(sizeof(BlockMapReference), .comparator = lstComparatorBlockMapReference);
|
||||
@ -317,6 +338,7 @@ blockMapWrite(const BlockMap *const this, IoWrite *const output, const bool bloc
|
||||
const BlockMapReference referenceAdd =
|
||||
{
|
||||
.reference = reference->reference,
|
||||
.superBlockSize = blockSize,
|
||||
.bundleId = reference->bundleId,
|
||||
.offset = reference->offset,
|
||||
};
|
||||
@ -335,7 +357,7 @@ blockMapWrite(const BlockMap *const this, IoWrite *const output, const bool bloc
|
||||
// the last one for the reference.
|
||||
if (reference->offset == referenceData->offset)
|
||||
{
|
||||
ASSERT(!blockEqual);
|
||||
ASSERT(reference->superBlockSize == referenceData->superBlockSize);
|
||||
|
||||
referenceEncoded |= BLOCK_MAP_FLAG_CONTINUE;
|
||||
referenceContinue = true;
|
||||
@ -374,9 +396,21 @@ blockMapWrite(const BlockMap *const this, IoWrite *const output, const bool bloc
|
||||
}
|
||||
}
|
||||
|
||||
// If block offset or total need to be stored then add the flag
|
||||
const unsigned int blockTotal = superBlockIdx - blockIdx;
|
||||
ASSERT(blockTotal > 0);
|
||||
|
||||
if (referenceContinue || superBlock->block != 0 ||
|
||||
blockTotal != superBlock->superBlockSize / blockSize + (superBlock->superBlockSize % blockSize == 0 ? 0 : 1))
|
||||
{
|
||||
superBlockEncoded |= BLOCK_MAP_FLAG_SUPER_BLOCK_TOTAL_OFFSET;
|
||||
}
|
||||
|
||||
// Write the continued reference now that we know if this will be the last super block in the reference
|
||||
if (referenceContinue)
|
||||
{
|
||||
ASSERT(superBlock->superBlockSize == referenceData->superBlockSize);
|
||||
|
||||
if (superBlockEncoded & BLOCK_MAP_FLAG_LAST)
|
||||
referenceEncoded |= BLOCK_MAP_FLAG_CONTINUE_LAST;
|
||||
|
||||
@ -386,50 +420,76 @@ blockMapWrite(const BlockMap *const this, IoWrite *const output, const bool bloc
|
||||
// Else write the super block size for the reference
|
||||
else
|
||||
{
|
||||
// If this is the first size written then just write the size
|
||||
if (sizeLast == 0)
|
||||
{
|
||||
ioWriteVarIntU64(output, superBlockEncoded | superBlock->size << BLOCK_MAP_SUPER_BLOCK_SHIFT);
|
||||
}
|
||||
// Else write the difference from the prior size. This depends on the expectation that the compressed size of
|
||||
// equal-sized blocks will be similar in order to be most efficient.
|
||||
else
|
||||
{
|
||||
ioWriteVarIntU64(
|
||||
output,
|
||||
superBlockEncoded | cvtInt64ToZigZag((int64_t)superBlock->size - sizeLast) << BLOCK_MAP_SUPER_BLOCK_SHIFT);
|
||||
}
|
||||
|
||||
// Set offset, size, and block for the super block
|
||||
referenceData->offset = superBlock->offset;
|
||||
referenceData->size = superBlock->size;
|
||||
referenceData->block = 0;
|
||||
|
||||
// If the super block size has changed then add the flag
|
||||
ASSERT(reference->superBlockSize > 0);
|
||||
|
||||
if (superBlock->superBlockSize != referenceData->superBlockSize)
|
||||
superBlockEncoded |= BLOCK_MAP_FLAG_SUPER_BLOCK_CHANGE;
|
||||
|
||||
// If this is the first size written then just write the size. Otherwise write the difference from the prior size.
|
||||
// This depends on the expectation that the compressed size of equal-sized blocks will be similar in order to be
|
||||
// most efficient.
|
||||
ioWriteVarIntU64(
|
||||
output,
|
||||
superBlockEncoded |
|
||||
(sizeLast == 0 ? superBlock->size : cvtInt64ToZigZag((int64_t)superBlock->size - sizeLast)) <<
|
||||
BLOCK_MAP_SUPER_BLOCK_SHIFT);
|
||||
|
||||
// If the super block size has changed then write it
|
||||
if (superBlockEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_CHANGE)
|
||||
{
|
||||
const uint64_t superBlockSizeEncoded =
|
||||
(superBlock->superBlockSize / blockSize) << BLOCK_MAP_SUPER_BLOCK_SIZE_SHIFT |
|
||||
(superBlock->superBlockSize % blockSize == 0 ? 0 : BLOCK_MAP_FLAG_SUPER_BLOCK_SIZE_REMAINDER);
|
||||
|
||||
ioWriteVarIntU64(output, superBlockSizeEncoded);
|
||||
|
||||
if (superBlockSizeEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_SIZE_REMAINDER)
|
||||
ioWriteVarIntU64(output, superBlock->superBlockSize % blockSize);
|
||||
|
||||
referenceData->superBlockSize = superBlock->superBlockSize;
|
||||
}
|
||||
}
|
||||
|
||||
sizeLast = (int64_t)superBlock->size;
|
||||
|
||||
// Write all blocks in the current super block in packed format
|
||||
while (blockIdx < superBlockIdx)
|
||||
// Write block total if the super block does not include all blocks with no block offset
|
||||
if (superBlockEncoded & BLOCK_MAP_FLAG_SUPER_BLOCK_TOTAL_OFFSET)
|
||||
{
|
||||
const BlockMapItem *const block = blockMapGet(this, blockIdx);
|
||||
ASSERT(block->block >= referenceData->block);
|
||||
// Write total blocks in the super block
|
||||
const uint64_t blockTotalEncoded =
|
||||
(blockTotal - 1) << BLOCK_MAP_BLOCK_TOTAL_SHIFT |
|
||||
(superBlock->block - referenceData->block > 0 ? BLOCK_MAP_FLAG_BLOCK_TOTAL_OFFSET : 0);
|
||||
|
||||
// Block no only needs to be written when super block size does not equal block size
|
||||
if (!blockEqual)
|
||||
ioWriteVarIntU64(output, blockTotalEncoded);
|
||||
|
||||
// If there is a gap in block no from the prior super block. This can happen when a super block is continued or
|
||||
// had blocks at the beginning overridden by a newer super block.
|
||||
if (blockTotalEncoded & BLOCK_MAP_FLAG_BLOCK_TOTAL_OFFSET)
|
||||
{
|
||||
// Write block no as the difference from the prior block
|
||||
ioWriteVarIntU64(
|
||||
output,
|
||||
(blockIdx == superBlockIdx - 1 ? BLOCK_MAP_FLAG_LAST : 0) |
|
||||
(block->block - referenceData->block) << BLOCK_MAP_BLOCK_SHIFT);
|
||||
|
||||
referenceData->block = block->block;
|
||||
ioWriteVarIntU64(output, superBlock->block - referenceData->block);
|
||||
referenceData->block = superBlock->block;
|
||||
}
|
||||
}
|
||||
|
||||
// Write checksum
|
||||
ioWrite(output, BUF(block->checksum, checksumSize));
|
||||
ASSERT(superBlock->block >= referenceData->block);
|
||||
|
||||
blockIdx++;
|
||||
// Increment reference block by number of blocks written
|
||||
referenceData->block += blockTotal;
|
||||
|
||||
// Write checksums
|
||||
for (; blockIdx < superBlockIdx; blockIdx++)
|
||||
{
|
||||
ASSERT(
|
||||
superBlock == blockMapGet(this, blockIdx) ||
|
||||
blockMapGet(this, blockIdx)->block == blockMapGet(this, blockIdx - 1)->block + 1);
|
||||
|
||||
ioWrite(output, BUF(blockMapGet(this, blockIdx)->checksum, checksumSize));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -20,9 +20,10 @@ typedef struct BlockMap BlockMap;
|
||||
typedef struct BlockMapItem
|
||||
{
|
||||
unsigned int reference; // Reference to backup where the block is stored
|
||||
uint64_t superBlockSize; // Super block size
|
||||
uint64_t bundleId; // Bundle where the block is stored (0 if not bundled)
|
||||
uint64_t offset; // Offset of super block into the bundle
|
||||
uint64_t size; // Size of the super block (including compression, etc.)
|
||||
uint64_t size; // Stored super block size (with compression, etc.)
|
||||
uint64_t block; // Block no inside of super block
|
||||
unsigned char checksum[XX_HASH_SIZE_MAX]; // Checksum of the block
|
||||
} BlockMapItem;
|
||||
@ -38,7 +39,7 @@ blockMapNew(void)
|
||||
}
|
||||
|
||||
// New block map from IO
|
||||
FN_EXTERN BlockMap *blockMapNewRead(IoRead *map, size_t checksumSize);
|
||||
FN_EXTERN BlockMap *blockMapNewRead(IoRead *map, size_t blockSize, size_t checksumSize);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Functions
|
||||
@ -52,7 +53,7 @@ blockMapAdd(BlockMap *const this, const BlockMapItem *const item)
|
||||
}
|
||||
|
||||
// Write map to IO
|
||||
FN_EXTERN void blockMapWrite(const BlockMap *this, IoWrite *output, bool blockEqual, size_t checksumSize);
|
||||
FN_EXTERN void blockMapWrite(const BlockMap *this, IoWrite *output, size_t blockSize, size_t checksumSize);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Getters/Setters
|
||||
|
||||
@ -15,7 +15,8 @@ Object type
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct BlockDeltaSuperBlock
|
||||
{
|
||||
uint64_t size; // Size of super block
|
||||
uint64_t superBlockSize; // Super block size
|
||||
uint64_t size; // Stored size of superblock (with compression, etc.)
|
||||
List *blockList; // Block list
|
||||
} BlockDeltaSuperBlock;
|
||||
|
||||
@ -37,10 +38,11 @@ struct BlockDelta
|
||||
|
||||
const BlockDeltaSuperBlock *superBlockData; // Current super block data
|
||||
unsigned int superBlockIdx; // Current super block index
|
||||
unsigned int blockNo; // Block number in current super block
|
||||
IoRead *chunkedRead; // Chunked read for current super block
|
||||
const BlockDeltaBlock *blockData; // Current block data
|
||||
unsigned int blockIdx; // Current block index
|
||||
unsigned int blockTotal; // Block total for super block
|
||||
unsigned int blockFindIdx; // Index of the block to find in the super block
|
||||
|
||||
BlockDeltaWrite write; // Block/offset to be returned for write
|
||||
};
|
||||
@ -173,6 +175,7 @@ blockDeltaNew(
|
||||
{
|
||||
BlockDeltaSuperBlock blockDeltaSuperBlockNew =
|
||||
{
|
||||
.superBlockSize = blockMapItem->superBlockSize,
|
||||
.size = blockMapItem->size,
|
||||
.blockList = lstNewP(sizeof(BlockDeltaBlock)),
|
||||
};
|
||||
@ -253,84 +256,65 @@ blockDeltaNext(BlockDelta *const this, const BlockDeltaRead *const readDelta, Io
|
||||
|
||||
// Set block info
|
||||
this->blockIdx = 0;
|
||||
this->blockNo = 0;
|
||||
this->blockData = lstGet(this->superBlockData->blockList, this->blockIdx);
|
||||
this->blockFindIdx = 0;
|
||||
this->blockTotal =
|
||||
(unsigned int)(this->superBlockData->superBlockSize / this->blockSize) +
|
||||
(this->superBlockData->superBlockSize % this->blockSize == 0 ? 0 : 1);
|
||||
this->blockData = lstGet(this->superBlockData->blockList, this->blockFindIdx);
|
||||
}
|
||||
|
||||
// Read encoded info about the block
|
||||
uint64_t blockEncoded = ioReadVarIntU64(this->chunkedRead);
|
||||
|
||||
// Loop through blocks in the super block until a match with the map is found
|
||||
do
|
||||
// Find required blocks in the super block
|
||||
while (this->blockIdx < this->blockTotal)
|
||||
{
|
||||
// Break out if encoded block info is zero and this is not the first block. This means the super block has ended when
|
||||
// all blocks are equal size.
|
||||
if (this->blockNo != 0 && blockEncoded == 0)
|
||||
{
|
||||
this->superBlockData = NULL;
|
||||
this->superBlockIdx++;
|
||||
break;
|
||||
}
|
||||
// Read encoded info about the block, which is not used here
|
||||
ioReadVarIntU64(this->chunkedRead);
|
||||
|
||||
// Apply block size limit if required and read the block
|
||||
bufUsedSet(this->write.block, 0);
|
||||
|
||||
if (blockEncoded & BLOCK_INCR_FLAG_SIZE)
|
||||
{
|
||||
size_t size = (size_t)ioReadVarIntU64(this->chunkedRead);
|
||||
bufLimitSet(this->write.block, size);
|
||||
}
|
||||
if (this->blockIdx == this->blockTotal - 1 && this->superBlockData->superBlockSize % this->blockSize != 0)
|
||||
bufLimitSet(this->write.block, (size_t)(this->superBlockData->superBlockSize % this->blockSize));
|
||||
else
|
||||
bufLimitClear(this->write.block);
|
||||
|
||||
ioRead(this->chunkedRead, this->write.block);
|
||||
|
||||
// If the block matches the block we are expecting
|
||||
if (this->blockNo == this->blockData->no)
|
||||
if (this->blockIdx == this->blockData->no)
|
||||
{
|
||||
ASSERT(result == NULL);
|
||||
|
||||
this->write.offset = this->blockData->offset;
|
||||
result = &this->write;
|
||||
this->blockIdx++;
|
||||
this->blockFindIdx++;
|
||||
|
||||
// Get the next block if there are any more to read
|
||||
if (this->blockIdx < lstSize(this->superBlockData->blockList))
|
||||
{
|
||||
this->blockData = lstGet(this->superBlockData->blockList, this->blockIdx);
|
||||
if (this->blockFindIdx < lstSize(this->superBlockData->blockList))
|
||||
this->blockData = lstGet(this->superBlockData->blockList, this->blockFindIdx);
|
||||
}
|
||||
// Else stop processing if this is the last super block. This is only works for the last super block because
|
||||
// otherwise the blocks must be read sequentially.
|
||||
else if (this->superBlockIdx >= lstSize(readDelta->superBlockList) - 1)
|
||||
{
|
||||
this->superBlockData = NULL;
|
||||
this->superBlockIdx++;
|
||||
}
|
||||
}
|
||||
|
||||
// If the size was set (meaning this is the last block in the super block) it must also be the last block we are looking
|
||||
// for in the read. Partial blocks cannot happen in the middle of a read and there is code above to cut out early on the
|
||||
// last super block of a read when possible.
|
||||
ASSERT(blockEncoded ^ BLOCK_INCR_FLAG_SIZE || this->superBlockIdx >= lstSize(readDelta->superBlockList) - 1);
|
||||
|
||||
// Increment the block to read in the super block
|
||||
this->blockNo++;
|
||||
this->blockIdx++;
|
||||
|
||||
// Break if there is a result
|
||||
if (result != NULL)
|
||||
break;
|
||||
|
||||
// Read encoded info about the block
|
||||
blockEncoded = ioReadVarIntU64(this->chunkedRead);
|
||||
}
|
||||
while (true);
|
||||
|
||||
// Break if there is a result
|
||||
if (result != NULL)
|
||||
break;
|
||||
|
||||
this->superBlockData = NULL;
|
||||
this->superBlockIdx++;
|
||||
}
|
||||
|
||||
// If no result then the super blocks have been read. Reset for the next read.
|
||||
if (result == NULL)
|
||||
{
|
||||
ASSERT(this->superBlockIdx == lstSize(readDelta->superBlockList));
|
||||
ASSERT(this->blockIdx == this->blockTotal);
|
||||
|
||||
this->superBlockData = NULL;
|
||||
this->superBlockIdx = 0;
|
||||
}
|
||||
|
||||
@ -295,7 +295,8 @@ restoreFile(
|
||||
|
||||
// Read block map. This will be compared to the block checksum list already created to determine which
|
||||
// blocks need to be fetched from the repository. If we got here there must be at least one block to fetch.
|
||||
const BlockMap *const blockMap = blockMapNewRead(storageReadIo(repoFileRead), file->blockIncrChecksumSize);
|
||||
const BlockMap *const blockMap = blockMapNewRead(
|
||||
storageReadIo(repoFileRead), file->blockIncrSize, file->blockIncrChecksumSize);
|
||||
|
||||
// The repo file needs to be closed so that block lists can be read from the remote protocol
|
||||
ioReadClose(storageReadIo(repoFileRead));
|
||||
@ -342,6 +343,8 @@ restoreFile(
|
||||
|
||||
deltaWrite = blockDeltaNext(blockDelta, read, storageReadIo(superBlockRead));
|
||||
}
|
||||
|
||||
storageReadFree(superBlockRead);
|
||||
}
|
||||
|
||||
// Close the file to complete the update
|
||||
|
||||
@ -50,7 +50,8 @@ testBlockDelta(const BlockMap *const blockMap, const size_t blockSize, const siz
|
||||
{
|
||||
const BlockDeltaSuperBlock *const superBlock = lstGet(read->superBlockList, superBlockIdx);
|
||||
|
||||
strCatFmt(result, " super block {size: %" PRIu64 "}\n", superBlock->size);
|
||||
strCatFmt(
|
||||
result, " super block {max: %" PRIu64 ", size: %" PRIu64 "}\n", superBlock->superBlockSize, superBlock->size);
|
||||
|
||||
for (unsigned int blockIdx = 0; blockIdx < lstSize(superBlock->blockList); blockIdx++)
|
||||
{
|
||||
@ -205,7 +206,8 @@ testBackupValidateList(
|
||||
|
||||
ioReadOpen(storageReadIo(read));
|
||||
|
||||
const BlockMap *const blockMap = blockMapNewRead(storageReadIo(read), file.blockIncrChecksumSize);
|
||||
const BlockMap *const blockMap = blockMapNewRead(
|
||||
storageReadIo(read), file.blockIncrSize, file.blockIncrChecksumSize);
|
||||
|
||||
// Build map log
|
||||
String *const mapLog = strNew();
|
||||
@ -682,6 +684,7 @@ testRun(void)
|
||||
BlockMapItem blockMapItem =
|
||||
{
|
||||
.reference = 128,
|
||||
.superBlockSize = 1,
|
||||
.bundleId = 0,
|
||||
.offset = 0,
|
||||
.size = 3,
|
||||
@ -694,6 +697,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 128,
|
||||
.superBlockSize = 1,
|
||||
.bundleId = 0,
|
||||
.offset = 3,
|
||||
.size = 5,
|
||||
@ -705,6 +709,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 0,
|
||||
.superBlockSize = 1,
|
||||
.bundleId = 1,
|
||||
.offset = 1,
|
||||
.size = 5,
|
||||
@ -716,6 +721,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 128,
|
||||
.superBlockSize = 1,
|
||||
.bundleId = 0,
|
||||
.offset = 8,
|
||||
.size = 99,
|
||||
@ -727,6 +733,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 0,
|
||||
.superBlockSize = 1,
|
||||
.bundleId = 1,
|
||||
.offset = 7,
|
||||
.size = 99,
|
||||
@ -739,6 +746,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 4,
|
||||
.superBlockSize = 1,
|
||||
.bundleId = 0,
|
||||
.offset = 0,
|
||||
.size = 8,
|
||||
@ -752,36 +760,36 @@ testRun(void)
|
||||
|
||||
Buffer *buffer = bufNew(256);
|
||||
IoWrite *write = ioBufferWriteNewOpen(buffer);
|
||||
TEST_RESULT_VOID(blockMapWrite(blockMap, write, true, 5), "save");
|
||||
TEST_RESULT_VOID(blockMapWrite(blockMap, write, 1, 5), "save");
|
||||
ioWriteClose(write);
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
strNewEncode(encodingHex, buffer),
|
||||
"02" // Blocks are equal
|
||||
"00" // Version 0
|
||||
|
||||
"8008" // reference 128
|
||||
"06" // super block size 3
|
||||
"18" // size 3
|
||||
"eeee01ffff" // checksum
|
||||
"09" // super block size 5
|
||||
"21" // size
|
||||
"eeee02ffff" // checksum
|
||||
|
||||
"06" // reference 0
|
||||
"01" // bundle 1
|
||||
"01" // offset 1
|
||||
"01" // super block size 5
|
||||
"01" // size 1
|
||||
"eeee03ffff" // checksum
|
||||
|
||||
"8008" // reference 128
|
||||
"f902" // super block size 99
|
||||
"e10b" // size 99
|
||||
"eeee04ffff" // checksum
|
||||
|
||||
"04" // reference 0
|
||||
"01" // offset 7
|
||||
"01" // super block size 99
|
||||
"01" // size 99
|
||||
"eeee05ffff" // checksum
|
||||
|
||||
"21" // reference 0
|
||||
"eb02" // super block size 8
|
||||
"a90b" // size 8
|
||||
"eeee88ffff", // checksum
|
||||
"compare");
|
||||
|
||||
@ -790,7 +798,7 @@ testRun(void)
|
||||
|
||||
Buffer *bufferCompare = bufNew(256);
|
||||
write = ioBufferWriteNewOpen(bufferCompare);
|
||||
TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer), 5), write, true, 5), "read and save");
|
||||
TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer), 1, 5), write, 1, 5), "read and save");
|
||||
ioWriteClose(write);
|
||||
|
||||
TEST_RESULT_STR(strNewEncode(encodingHex, bufferCompare), strNewEncode(encodingHex, buffer), "compare");
|
||||
@ -799,23 +807,23 @@ testRun(void)
|
||||
TEST_TITLE("equal block delta");
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer), 5), 8, 5),
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer), 1, 5), 1, 5),
|
||||
"read {reference: 128, bundleId: 0, offset: 0, size: 107}\n"
|
||||
" super block {size: 3}\n"
|
||||
" super block {max: 1, size: 3}\n"
|
||||
" block {no: 0, offset: 0}\n"
|
||||
" super block {size: 5}\n"
|
||||
" block {no: 0, offset: 8}\n"
|
||||
" super block {size: 99}\n"
|
||||
" block {no: 0, offset: 24}\n"
|
||||
" super block {max: 1, size: 5}\n"
|
||||
" block {no: 0, offset: 1}\n"
|
||||
" super block {max: 1, size: 99}\n"
|
||||
" block {no: 0, offset: 3}\n"
|
||||
"read {reference: 4, bundleId: 0, offset: 0, size: 8}\n"
|
||||
" super block {size: 8}\n"
|
||||
" block {no: 0, offset: 40}\n"
|
||||
" super block {max: 1, size: 8}\n"
|
||||
" block {no: 0, offset: 5}\n"
|
||||
"read {reference: 0, bundleId: 1, offset: 1, size: 5}\n"
|
||||
" super block {size: 5}\n"
|
||||
" block {no: 0, offset: 16}\n"
|
||||
" super block {max: 1, size: 5}\n"
|
||||
" block {no: 0, offset: 2}\n"
|
||||
"read {reference: 0, bundleId: 1, offset: 7, size: 99}\n"
|
||||
" super block {size: 99}\n"
|
||||
" block {no: 0, offset: 32}\n",
|
||||
" super block {max: 1, size: 99}\n"
|
||||
" block {no: 0, offset: 4}\n",
|
||||
"check delta");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
@ -826,6 +834,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 0,
|
||||
.superBlockSize = 6,
|
||||
.offset = 0,
|
||||
.size = 4,
|
||||
.block = 0,
|
||||
@ -837,6 +846,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 0,
|
||||
.superBlockSize = 6,
|
||||
.offset = 0,
|
||||
.size = 4,
|
||||
.block = 1,
|
||||
@ -848,6 +858,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 0,
|
||||
.superBlockSize = 6,
|
||||
.offset = 4,
|
||||
.size = 5,
|
||||
.block = 0,
|
||||
@ -859,6 +870,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 1,
|
||||
.superBlockSize = 3,
|
||||
.offset = 0,
|
||||
.size = 99,
|
||||
.block = 0,
|
||||
@ -870,6 +882,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 0,
|
||||
.superBlockSize = 6,
|
||||
.offset = 4,
|
||||
.size = 5,
|
||||
.block = 3,
|
||||
@ -881,6 +894,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 2,
|
||||
.superBlockSize = 2,
|
||||
.offset = 0,
|
||||
.size = 1,
|
||||
.block = 0,
|
||||
@ -892,6 +906,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 0,
|
||||
.superBlockSize = 6,
|
||||
.offset = 4,
|
||||
.size = 5,
|
||||
.block = 5,
|
||||
@ -903,6 +918,7 @@ testRun(void)
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 0,
|
||||
.superBlockSize = 2,
|
||||
.offset = 9,
|
||||
.size = 6,
|
||||
.block = 0,
|
||||
@ -911,12 +927,24 @@ testRun(void)
|
||||
|
||||
TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
|
||||
|
||||
blockMapItem = (BlockMapItem)
|
||||
{
|
||||
.reference = 1,
|
||||
.superBlockSize = 3,
|
||||
.offset = 99,
|
||||
.size = 1,
|
||||
.block = 1,
|
||||
.checksum = {0xee, 0xee, 0x09, 0, 0, 0, 0xff, 0xff},
|
||||
};
|
||||
|
||||
TEST_RESULT_VOID(blockMapAdd(blockMap, &blockMapItem), "add");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("write unequal block map");
|
||||
|
||||
buffer = bufNew(256);
|
||||
write = ioBufferWriteNewOpen(buffer);
|
||||
TEST_RESULT_VOID(blockMapWrite(blockMap, write, false, 8), "save");
|
||||
TEST_RESULT_VOID(blockMapWrite(blockMap, write, 3, 8), "save");
|
||||
ioWriteClose(write);
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
@ -924,37 +952,43 @@ testRun(void)
|
||||
"00" // Blocks are unequal
|
||||
|
||||
"00" // reference 0
|
||||
"08" // size 4
|
||||
"00" // block 0
|
||||
"22" // size 4
|
||||
"04" // super block size 6
|
||||
"eeee01000000ffff" // checksum
|
||||
|
||||
"03" // block 1
|
||||
"eeee02000000ffff" // checksum
|
||||
|
||||
"05" // size 5
|
||||
"01" // block 0
|
||||
"15" // size 5
|
||||
"00" // block total 1
|
||||
"eeee03000000ffff" // checksum
|
||||
|
||||
"08" // reference 1
|
||||
"f902" // size 99
|
||||
"01" // block 0
|
||||
"e10b" // size 99
|
||||
"eeee04000000ffff" // checksum
|
||||
|
||||
"06" // reference 0
|
||||
"07" // block 3
|
||||
"01" // block total 1
|
||||
"02" // block 3
|
||||
"eeee05000000ffff" // checksum
|
||||
|
||||
"10" // reference 2
|
||||
"0f" // size 1
|
||||
"01" // block 0
|
||||
"3b01" // size 1
|
||||
"02" // super block size 2
|
||||
"eeee06000000ffff" // checksum
|
||||
|
||||
"03" // reference 0
|
||||
"05" // size 5
|
||||
"02" // reference 0
|
||||
"01" // block total 1
|
||||
"01" // block 5
|
||||
"eeee07000000ffff" // checksum
|
||||
"05" // size 6
|
||||
"01" // block
|
||||
"eeee08000000ffff", // checksum
|
||||
|
||||
"13" // size 6
|
||||
"0102" // size 2
|
||||
"eeee08000000ffff" // checksum
|
||||
|
||||
"09" // reference 1
|
||||
"4d" // size 1
|
||||
"01" // block total 1
|
||||
"01" // block 1
|
||||
"eeee09000000ffff", // checksum
|
||||
"compare");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
@ -962,7 +996,7 @@ testRun(void)
|
||||
|
||||
bufferCompare = bufNew(256);
|
||||
write = ioBufferWriteNewOpen(bufferCompare);
|
||||
TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer), 8), write, false, 8), "read and save");
|
||||
TEST_RESULT_VOID(blockMapWrite(blockMapNewRead(ioBufferReadNewOpen(buffer), 3, 8), write, 3, 8), "read and save");
|
||||
ioWriteClose(write);
|
||||
|
||||
TEST_RESULT_STR(strNewEncode(encodingHex, bufferCompare), strNewEncode(encodingHex, buffer), "compare");
|
||||
@ -971,23 +1005,25 @@ testRun(void)
|
||||
TEST_TITLE("unequal block delta");
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer), 8), 8, 8),
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(buffer), 3, 8), 3, 8),
|
||||
"read {reference: 2, bundleId: 0, offset: 0, size: 1}\n"
|
||||
" super block {size: 1}\n"
|
||||
" block {no: 0, offset: 40}\n"
|
||||
"read {reference: 1, bundleId: 0, offset: 0, size: 99}\n"
|
||||
" super block {size: 99}\n"
|
||||
" block {no: 0, offset: 24}\n"
|
||||
" super block {max: 2, size: 1}\n"
|
||||
" block {no: 0, offset: 15}\n"
|
||||
"read {reference: 1, bundleId: 0, offset: 0, size: 100}\n"
|
||||
" super block {max: 3, size: 99}\n"
|
||||
" block {no: 0, offset: 9}\n"
|
||||
" super block {max: 3, size: 1}\n"
|
||||
" block {no: 1, offset: 24}\n"
|
||||
"read {reference: 0, bundleId: 0, offset: 0, size: 15}\n"
|
||||
" super block {size: 4}\n"
|
||||
" super block {max: 6, size: 4}\n"
|
||||
" block {no: 0, offset: 0}\n"
|
||||
" block {no: 1, offset: 8}\n"
|
||||
" super block {size: 5}\n"
|
||||
" block {no: 0, offset: 16}\n"
|
||||
" block {no: 3, offset: 32}\n"
|
||||
" block {no: 5, offset: 48}\n"
|
||||
" super block {size: 6}\n"
|
||||
" block {no: 0, offset: 56}\n",
|
||||
" block {no: 1, offset: 3}\n"
|
||||
" super block {max: 6, size: 5}\n"
|
||||
" block {no: 0, offset: 6}\n"
|
||||
" block {no: 3, offset: 12}\n"
|
||||
" block {no: 5, offset: 18}\n"
|
||||
" super block {max: 2, size: 6}\n"
|
||||
" block {no: 0, offset: 21}\n",
|
||||
"check delta");
|
||||
}
|
||||
|
||||
@ -1042,19 +1078,19 @@ testRun(void)
|
||||
|
||||
uint64_t mapSize;
|
||||
TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size");
|
||||
TEST_RESULT_UINT(mapSize, 11, "map size");
|
||||
TEST_RESULT_UINT(mapSize, 13, "map size");
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
|
||||
"02010201313200", // block 0
|
||||
"020031023200", // block 0
|
||||
"block list");
|
||||
|
||||
const Buffer *map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
|
||||
"read {reference: 0, bundleId: 0, offset: 0, size: 7}\n"
|
||||
" super block {size: 7}\n"
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 3, 8), 3, 8),
|
||||
"read {reference: 0, bundleId: 0, offset: 0, size: 6}\n"
|
||||
" super block {max: 2, size: 6}\n"
|
||||
" block {no: 0, offset: 0}\n",
|
||||
"check delta");
|
||||
|
||||
@ -1068,7 +1104,7 @@ testRun(void)
|
||||
TEST_RESULT_VOID(
|
||||
ioFilterGroupAdd(
|
||||
ioWriteFilterGroup(write),
|
||||
blockIncrNewPack(ioFilterParamList(blockIncrNew(5, 3, 8, 2, 4, 5, NULL, NULL, NULL)))),
|
||||
blockIncrNewPack(ioFilterParamList(blockIncrNew(2, 3, 8, 2, 4, 5, NULL, NULL, NULL)))),
|
||||
"block incr");
|
||||
TEST_RESULT_VOID(ioWriteOpen(write), "open");
|
||||
TEST_RESULT_VOID(ioWrite(write, source), "write");
|
||||
@ -1079,21 +1115,21 @@ testRun(void)
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
|
||||
"020041014243020000" // block 0
|
||||
"02025801595a020000" // block 1
|
||||
"020231013233020000", // block 2
|
||||
"02004101424300" // block 0
|
||||
"02015801595a00" // block 1
|
||||
"02013101323300", // block 2
|
||||
"block list");
|
||||
|
||||
map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
|
||||
"read {reference: 2, bundleId: 4, offset: 5, size: 27}\n"
|
||||
" super block {size: 9}\n"
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 3, 8), 3, 8),
|
||||
"read {reference: 2, bundleId: 4, offset: 5, size: 21}\n"
|
||||
" super block {max: 3, size: 7}\n"
|
||||
" block {no: 0, offset: 0}\n"
|
||||
" super block {size: 9}\n"
|
||||
" super block {max: 3, size: 7}\n"
|
||||
" block {no: 0, offset: 3}\n"
|
||||
" super block {size: 9}\n"
|
||||
" super block {max: 3, size: 7}\n"
|
||||
" block {no: 0, offset: 6}\n",
|
||||
"check delta");
|
||||
|
||||
@ -1115,27 +1151,27 @@ testRun(void)
|
||||
TEST_RESULT_VOID(ioWriteClose(write), "close");
|
||||
|
||||
TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size");
|
||||
TEST_RESULT_UINT(mapSize, 42, "map size");
|
||||
TEST_RESULT_UINT(mapSize, 44, "map size");
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
|
||||
"0300414302430000" // block 0
|
||||
"0307014000", // block 3
|
||||
"03004143044300" // block 0
|
||||
"02034000", // block 3
|
||||
"block list");
|
||||
|
||||
map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
|
||||
"read {reference: 3, bundleId: 0, offset: 0, size: 13}\n"
|
||||
" super block {size: 8}\n"
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 3, 8), 3, 8),
|
||||
"read {reference: 3, bundleId: 0, offset: 0, size: 11}\n"
|
||||
" super block {max: 3, size: 7}\n"
|
||||
" block {no: 0, offset: 0}\n"
|
||||
" super block {size: 5}\n"
|
||||
" super block {max: 1, size: 4}\n"
|
||||
" block {no: 0, offset: 9}\n"
|
||||
"read {reference: 2, bundleId: 4, offset: 14, size: 18}\n"
|
||||
" super block {size: 9}\n"
|
||||
"read {reference: 2, bundleId: 4, offset: 12, size: 14}\n"
|
||||
" super block {max: 3, size: 7}\n"
|
||||
" block {no: 0, offset: 3}\n"
|
||||
" super block {size: 9}\n"
|
||||
" super block {max: 3, size: 7}\n"
|
||||
" block {no: 0, offset: 6}\n",
|
||||
"check delta");
|
||||
|
||||
@ -1157,24 +1193,24 @@ testRun(void)
|
||||
TEST_RESULT_VOID(ioWriteClose(write), "close");
|
||||
|
||||
TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size");
|
||||
TEST_RESULT_UINT(mapSize, 33, "map size");
|
||||
TEST_RESULT_UINT(mapSize, 32, "map size");
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
strNewEncode(encodingHex, BUF(bufPtr(destination), bufUsed(destination) - (size_t)mapSize)),
|
||||
"020041014243" // super block 0 / block 0
|
||||
"01025801595a020000" // super block 0 / block 1
|
||||
"020231013233020000", // block 2
|
||||
"01015801595a00" // super block 0 / block 1
|
||||
"02013101323300", // block 2
|
||||
"block list");
|
||||
|
||||
map = BUF(bufPtr(destination) + (bufUsed(destination) - (size_t)mapSize), (size_t)mapSize);
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 8), 3, 8),
|
||||
"read {reference: 2, bundleId: 4, offset: 5, size: 24}\n"
|
||||
" super block {size: 15}\n"
|
||||
testBlockDelta(blockMapNewRead(ioBufferReadNewOpen(map), 3, 8), 3, 8),
|
||||
"read {reference: 2, bundleId: 4, offset: 5, size: 20}\n"
|
||||
" super block {max: 6, size: 13}\n"
|
||||
" block {no: 0, offset: 0}\n"
|
||||
" block {no: 1, offset: 3}\n"
|
||||
" super block {size: 9}\n"
|
||||
" super block {max: 3, size: 7}\n"
|
||||
" block {no: 0, offset: 6}\n",
|
||||
"check delta");
|
||||
|
||||
@ -3653,7 +3689,7 @@ testRun(void)
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/0, 16.0KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16383, 8KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24575, 16KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (bundle 1/40998, 2B, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/PG_VERSION (bundle 1/40996, 2B, [PCT]) checksum [SHA1]\n"
|
||||
"P00 INFO: execute non-exclusive backup stop and wait for all WAL segments to archive\n"
|
||||
"P00 INFO: backup stop archive = 0000000105DBF06000000001, lsn = 5dbf060/300000\n"
|
||||
"P00 DETAIL: wrote 'backup_label' file returned from backup stop function\n"
|
||||
@ -3683,9 +3719,9 @@ testRun(void)
|
||||
",\"timestamp\":1572800000}\n"
|
||||
"pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
|
||||
",\"timestamp\":1572800002}\n"
|
||||
"pg_data/block-incr-grow={\"bi\":1,\"bim\":26,\"checksum\":\"ebdd38b69cd5b9f2d00d273c981e16960fbbb4f7\""
|
||||
"pg_data/block-incr-grow={\"bi\":1,\"bim\":24,\"checksum\":\"ebdd38b69cd5b9f2d00d273c981e16960fbbb4f7\""
|
||||
",\"size\":24576,\"timestamp\":1572800000}\n"
|
||||
"pg_data/block-incr-shrink={\"bi\":1,\"bim\":30,\"checksum\":\"ce5f8864058b1bb274244b512cb9641355987134\""
|
||||
"pg_data/block-incr-shrink={\"bi\":1,\"bim\":29,\"checksum\":\"ce5f8864058b1bb274244b512cb9641355987134\""
|
||||
",\"size\":16385,\"timestamp\":1572800000}\n"
|
||||
"pg_data/global/pg_control={\"size\":8192,\"timestamp\":1572800000}\n"
|
||||
"pg_data/grow-to-block-incr={\"checksum\":\"f5a5c308cf5fcb52bccebe2365f8ed56acbcc41d\",\"size\":16383"
|
||||
@ -3764,8 +3800,8 @@ testRun(void)
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-larger (1.4MB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-grow (128KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/0, 16KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16420, 8KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24612, 16.0KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16418, 8KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24610, 16.0KB, [PCT]) checksum [SHA1]\n"
|
||||
"P00 DETAIL: reference pg_data/PG_VERSION to 20191103-165320F\n"
|
||||
"P00 INFO: execute non-exclusive backup stop and wait for all WAL segments to archive\n"
|
||||
"P00 INFO: backup stop archive = 0000000105DC213000000001, lsn = 5dc2130/300000\n"
|
||||
@ -3796,14 +3832,14 @@ testRun(void)
|
||||
",\"size\":2,\"timestamp\":1572800000}\n"
|
||||
"pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
|
||||
",\"timestamp\":1573000002}\n"
|
||||
"pg_data/block-incr-grow={\"bi\":1,\"bim\":123,\"checksum\":\"1ddde8db92dd9019be0819ae4f9ad9cea2fae399\""
|
||||
"pg_data/block-incr-grow={\"bi\":1,\"bim\":114,\"checksum\":\"1ddde8db92dd9019be0819ae4f9ad9cea2fae399\""
|
||||
",\"size\":131072,\"timestamp\":1573000000}\n"
|
||||
"pg_data/block-incr-larger={\"bi\":8,\"bic\":7,\"bim\":194"
|
||||
"pg_data/block-incr-larger={\"bi\":8,\"bic\":7,\"bim\":173"
|
||||
",\"checksum\":\"eec53a6da79c00b3c658a7e09f44b3e9efefd960\",\"size\":1507328,\"timestamp\":1573000000}\n"
|
||||
"pg_data/block-incr-shrink={\"checksum\":\"1c6a17f67562d8b3f64f1b5f2ee592a4c2809b3b\",\"size\":16383"
|
||||
",\"timestamp\":1573000000}\n"
|
||||
"pg_data/global/pg_control={\"size\":8192,\"timestamp\":1573000000}\n"
|
||||
"pg_data/grow-to-block-incr={\"bi\":1,\"bim\":27,\"checksum\":\"4f560611d9dc9212432970e5c4bec15d876c226e\","
|
||||
"pg_data/grow-to-block-incr={\"bi\":1,\"bim\":26,\"checksum\":\"4f560611d9dc9212432970e5c4bec15d876c226e\","
|
||||
"\"size\":16385,\"timestamp\":1573000000}\n"
|
||||
"pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19"
|
||||
",\"timestamp\":1573000002}\n"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user