krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-07 01:10:04 +02:00
Code Issues Releases 1 Activity

Multi-block compression scaffolding - works on single-block files

Browse Source
This commit is contained in:
senhuang42 2020-10-29 11:01:04 -04:00
parent 75b01f34b9
commit 7eb6fa7be4
1 changed files with 59 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
lib/compress/zstd_compress.c
View File

@ -4609,46 +4609,76 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity,
ZSTD_CCtx* cctx, ZSTD_CCtx* cctx,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize, const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* src, size_t srcSize) { const void* src, size_t srcSize) {
U32 cSize; U32 cSize = 0;
U32 lastBlock; U32 lastBlock;
U32 blockSize;
U32 compressedSeqsSize;
size_t remaining = srcSize;
ZSTD_sequenceRange seqRange = {0, 0, 0, 0}; ZSTD_sequenceRange seqRange = {0, 0, 0, 0};
/* Derive the appropriate block size */ /* Derive the appropriate block size */
size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << cctx->appliedParams.cParams.windowLog), srcSize));
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
BYTE const* ip = (BYTE const*)src; BYTE const* ip = (BYTE const*)src;
BYTE* op = (BYTE*)dst; BYTE* op = (BYTE*)dst;
ZSTD_updateSequenceRange(&seqRange, srcSize, inSeqs, inSeqsSize); while (remaining) {
DEBUGLOG(4, "--NEW BLOCK--");
lastBlock = remaining <= cctx->blockSize;
blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize;
ZSTD_copySequencesToSeqStore(cctx, &seqRange, inSeqs, inSeqsSize, src, srcSize); DEBUGLOG(4, "blockSize: %u", blockSize);
/* Skip over uncompressible blocks */
if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
DEBUGLOG(4, "block too small: %u", blockSize);
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
ip += cctx->blockSize;
op += cSize;
remaining -= blockSize;
continue;
}
cSize = ZSTD_compressSequences(&cctx->seqStore, ZSTD_updateSequenceRange(&seqRange, blockSize, inSeqs, inSeqsSize);
&cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, ZSTD_copySequencesToSeqStore(cctx, &seqRange, inSeqs, inSeqsSize, ip, blockSize);
&cctx->appliedParams,
dst + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, compressedSeqsSize += ZSTD_compressSequences(&cctx->seqStore,
srcSize, &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy,
cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, &cctx->appliedParams,
cctx->bmi2); op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize,
if (ZSTD_isError(cSize)) { blockSize,
printf("ERROR\n"); cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
} else if (cSize == 0) { cctx->bmi2);
DEBUGLOG(4, "NO compress BLOCK"); DEBUGLOG(4, "Compressed sequences size : %u", compressedSeqsSize);
cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); if (ZSTD_isError(compressedSeqsSize)) {
return cSize; DEBUGLOG(4, "ERROR");
} return compressedSeqsSize;
DEBUGLOG(4, "Compressed sequences size : %u", cSize); } else if (compressedSeqsSize == 0) {
/* Error checking */ DEBUGLOG(4, "NO compress BLOCK");
if (!ZSTD_isError(cSize) && cSize > 1) { /* ZSTD_noCompressBlock writes block header as well */
ZSTD_confirmRepcodesAndEntropyTables(cctx); cSize += ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
} else {
cSize += compressedSeqsSize;
/* Error checking */
if (!ZSTD_isError(cSize) && cSize > 1) {
ZSTD_confirmRepcodesAndEntropyTables(cctx);
}
/* Write block header */
U32 const cBlockHeader = cSize == 1 ?
lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3):
lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
MEM_writeLE24(op, cBlockHeader);
cSize += ZSTD_blockHeaderSize;
}
DEBUGLOG(4, "cumulative cSize: %u", cSize);
if (lastBlock) {
DEBUGLOG(4, "reached last block, breaking");
break;
} else {
ip += blockSize;
op += cSize;
remaining -= blockSize;
}
} }
/* Write block header */
U32 const cBlockHeader = cSize == 1 ?
lastBlock + (((U32)bt_rle)<<1) + (U32)(cctx->blockSize << 3) :
lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
MEM_writeLE24(dst, cBlockHeader);
cSize += ZSTD_blockHeaderSize;
return cSize; return cSize;
} }