diff --git a/NEWS b/NEWS index f99b0376b..fff7bf04e 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,8 @@ v0.4.6 fix : fast compression mode on Windows Improved : high compression mode on repetitive data -Added : ZSTD_duplicateCCtx() +New : block-level API +New : ZSTD_duplicateCCtx() v0.4.5 new : -m/--multiple : compress/decompress multiple files diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 3cedeb797..e66fedbe5 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1794,10 +1794,9 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int } -size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit); - if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */ return blockCompressor(zc, dst, maxDstSize, src, srcSize); } @@ -1827,7 +1826,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr, if (ctxPtr->dictLimit < ctxPtr->lowLimit) ctxPtr->dictLimit = ctxPtr->lowLimit; } - cSize = ZSTD_compressBlock(ctxPtr, op+3, maxDstSize-3, ip, blockSize); + cSize = ZSTD_compressBlock_internal(ctxPtr, op+3, maxDstSize-3, ip, blockSize); if (ZSTD_isError(cSize)) return cSize; if (cSize == 0) @@ -1853,14 +1852,15 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr, } -size_t ZSTD_compressContinue (ZSTD_CCtx* zc, - void* dst, size_t dstSize, - const void* src, size_t srcSize) +static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc, + void* dst, size_t dstSize, + const void* src, size_t srcSize, + U32 frame) { const BYTE* const ip = (const BYTE*) src; size_t hbSize = 0; - if (zc->stage==0) + if (frame && (zc->stage==0)) { hbSize = zc->hbSize; if (dstSize <= hbSize) return ERROR(dstSize_tooSmall); @@ -1899,7 +1899,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, else zc->nextToUpdate -= correction; } - /* input-dictionary overlap */ + /* if input and dictionary overlap : reduce dictionary (presumed modified by input) */ if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit)) { zc->lowLimit = (U32)(ip + srcSize - zc->dictBase); @@ -1908,12 +1908,31 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, zc->nextSrc = ip + srcSize; { - size_t cSize = ZSTD_compress_generic (zc, dst, dstSize, src, srcSize); + size_t cSize; + if (frame) cSize = ZSTD_compress_generic (zc, dst, dstSize, src, srcSize); + else cSize = ZSTD_compressBlock_internal (zc, dst, dstSize, src, srcSize); if (ZSTD_isError(cSize)) return cSize; return cSize + hbSize; } } + +size_t ZSTD_compressContinue (ZSTD_CCtx* zc, + void* dst, size_t dstSize, + const void* src, size_t srcSize) +{ + return ZSTD_compressContinue_internal(zc, dst, dstSize, src, srcSize, 1); +} + + +size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong); + if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */ + return ZSTD_compressContinue_internal(zc, dst, maxDstSize, src, srcSize, 0); +} + + size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t srcSize) { const BYTE* const ip = (const BYTE*) src; diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 3431e327d..5dec85888 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -658,8 +658,7 @@ static size_t ZSTD_decompressSequences( } -static size_t ZSTD_decompressBlock( - ZSTD_DCtx* dctx, +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 62bd93577..f1e72e955 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -144,7 +144,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz Note that dictionary presence is a "hidden" information, the decoder needs to be aware that it is required for proper decoding, or decoding will fail. - If you want to compress multiple messages using same dictionary, + If you want to compress a lot of messages using same dictionary, it can be beneficial to duplicate compression context rather than reloading dictionary each time. In such case, use ZSTD_duplicateCCtx(), which will need an already created ZSTD_CCtx, in order to duplicate compression context into it. @@ -157,7 +157,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz Finish a frame with ZSTD_compressEnd(), which will write the epilogue. Without it, the frame will be considered incomplete by decoders. - You can then reuse ZSTD_CCtx to compress new frames. + You can then reuse ZSTD_CCtx to compress some new frame. */ @@ -196,9 +196,36 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ma It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. */ +/* ************************************** +* Block functions +****************************************/ + +/*!Block functions produce and decode raw zstd blocks, without frame metadata. + It saves associated header sizes. + But user will have to save and regenerate fields required to regenerate data, such as block sizes. + + A few rules to respect : + - Uncompressed block size must be <= 128 KB + - Compressing or decompressing require a context structure + + Use ZSTD_createXCtx() to create them + - It is necessary to init context before starting + + compression : ZSTD_compressBegin(), which allows selection of compression level or parameters + + decompression : ZSTD_resetDCtx() + + If you compress multiple blocks without resetting, next blocks will create references to previous ones + - Dictionary can optionally be inserted, using ZSTD_de/compress_insertDictionary() + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. + + User must test for such outcome and be able to deal with uncompressed data + + ZSTD_decompressBlock() doesn't accept uncompressed data as input +*/ + +size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); + + /* ************************************* * Pre-defined compression levels ***************************************/ diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 2d1d45151..28f50c54a 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -227,6 +227,7 @@ static int basicUnitTests(U32 seed, double compressibility) compressedBuffer, cSize, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; + if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error; ZSTD_freeCCtx(ctxOrig); /* if ctxOrig is read, will produce segfault */ DISPLAYLEVEL(4, "OK \n"); @@ -249,6 +250,7 @@ static int basicUnitTests(U32 seed, double compressibility) compressedBuffer, cSize, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; + if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error; ZSTD_freeDCtx(dctx); DISPLAYLEVEL(4, "OK \n"); } @@ -266,6 +268,32 @@ static int basicUnitTests(U32 seed, double compressibility) if (!ZSTD_isError(result)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); + /* block API tests */ + { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + const size_t blockSize = 100 KB; + + /* basic block compression */ + DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++); + result = ZSTD_compressBegin(cctx, 5); + if (ZSTD_isError(result)) goto _output_error; + cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); + if (ZSTD_isError(cSize)) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++); + result = ZSTD_resetDCtx(dctx); + if (ZSTD_isError(result)) goto _output_error; + result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); + if (ZSTD_isError(result)) goto _output_error; + if (result != blockSize) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + } + /* long rle test */ { size_t sampleSize = 0;