block-level API

NEWS

@@ -1,7 +1,8 @@
 v0.4.6
 fix : fast compression mode on Windows
 Improved : high compression mode on repetitive data
-Added : ZSTD_duplicateCCtx()
+New : block-level API
+New : ZSTD_duplicateCCtx()
 
 v0.4.5
 new : -m/--multiple : compress/decompress multiple files

lib/zstd_compress.c

@@ -1794,10 +1794,9 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int
 }
 
 
-size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
 {
     ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit);
-    if (srcSize < MIN_CBLOCK_SIZE+3) return 0;   /* don't even attempt compression below a certain srcSize */
     return blockCompressor(zc, dst, maxDstSize, src, srcSize);
 }
 
@@ -1827,7 +1826,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr,
             if (ctxPtr->dictLimit < ctxPtr->lowLimit) ctxPtr->dictLimit = ctxPtr->lowLimit;
         }
 
-        cSize = ZSTD_compressBlock(ctxPtr, op+3, maxDstSize-3, ip, blockSize);
+        cSize = ZSTD_compressBlock_internal(ctxPtr, op+3, maxDstSize-3, ip, blockSize);
         if (ZSTD_isError(cSize)) return cSize;
 
         if (cSize == 0)
@@ -1853,14 +1852,15 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr,
 }
 
 
-size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
-                                 void* dst, size_t dstSize,
-                           const void* src, size_t srcSize)
+static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
+                              void* dst, size_t dstSize,
+                        const void* src, size_t srcSize,
+                               U32 frame)
 {
     const BYTE* const ip = (const BYTE*) src;
     size_t hbSize = 0;
 
-    if (zc->stage==0)
+    if (frame && (zc->stage==0))
     {
         hbSize = zc->hbSize;
         if (dstSize <= hbSize) return ERROR(dstSize_tooSmall);
@@ -1899,7 +1899,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
         else zc->nextToUpdate -= correction;
     }
 
-    /* input-dictionary overlap */
+    /* if input and dictionary overlap : reduce dictionary (presumed modified by input) */
     if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit))
     {
         zc->lowLimit = (U32)(ip + srcSize - zc->dictBase);
@@ -1908,12 +1908,31 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
 
     zc->nextSrc = ip + srcSize;
     {
-        size_t cSize = ZSTD_compress_generic (zc, dst, dstSize, src, srcSize);
+        size_t cSize;
+        if (frame) cSize = ZSTD_compress_generic (zc, dst, dstSize, src, srcSize);
+        else cSize = ZSTD_compressBlock_internal (zc, dst, dstSize, src, srcSize);
         if (ZSTD_isError(cSize)) return cSize;
         return cSize + hbSize;
     }
 }
 
+
+size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
+                              void* dst, size_t dstSize,
+                        const void* src, size_t srcSize)
+{
+    return ZSTD_compressContinue_internal(zc, dst, dstSize, src, srcSize, 1);
+}
+
+
+size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+    if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong);
+    if (srcSize < MIN_CBLOCK_SIZE+3) return 0;   /* don't even attempt compression below a certain srcSize */
+    return ZSTD_compressContinue_internal(zc, dst, maxDstSize, src, srcSize, 0);
+}
+
+
 size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t srcSize)
 {
     const BYTE* const ip = (const BYTE*) src;

lib/zstd_decompress.c

@@ -658,8 +658,7 @@ static size_t ZSTD_decompressSequences(
 }
 
 
-static size_t ZSTD_decompressBlock(
-                            ZSTD_DCtx* dctx,
+size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
                             void* dst, size_t maxDstSize,
                       const void* src, size_t srcSize)
 {

lib/zstd_static.h

@@ -144,7 +144,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz
   Note that dictionary presence is a "hidden" information,
   the decoder needs to be aware that it is required for proper decoding, or decoding will fail.
 
-  If you want to compress multiple messages using same dictionary,
+  If you want to compress a lot of messages using same dictionary,
   it can be beneficial to duplicate compression context rather than reloading dictionary each time.
   In such case, use ZSTD_duplicateCCtx(), which will need an already created ZSTD_CCtx,
   in order to duplicate compression context into it.
@@ -157,7 +157,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz
   Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
   Without it, the frame will be considered incomplete by decoders.
 
-  You can then reuse ZSTD_CCtx to compress new frames.
+  You can then reuse ZSTD_CCtx to compress some new frame.
 */
 
 
@@ -196,9 +196,36 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ma
   It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
 
   A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
+  Context can then be reset to start a new decompression.
 */
 
 
+/* **************************************
+*  Block functions
+****************************************/
+
+/*!Block functions produce and decode raw zstd blocks, without frame metadata.
+   It saves associated header sizes.
+   But user will have to save and regenerate fields required to regenerate data, such as block sizes.
+
+   A few rules to respect :
+   - Uncompressed block size must be <= 128 KB
+   - Compressing or decompressing require a context structure
+     + Use ZSTD_createXCtx() to create them
+   - It is necessary to init context before starting
+     + compression : ZSTD_compressBegin(), which allows selection of compression level or parameters
+     + decompression : ZSTD_resetDCtx()
+     + If you compress multiple blocks without resetting, next blocks will create references to previous ones
+   - Dictionary can optionally be inserted, using ZSTD_de/compress_insertDictionary()
+   - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
+     + User must test for such outcome and be able to deal with uncompressed data
+     + ZSTD_decompressBlock() doesn't accept uncompressed data as input
+*/
+
+size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
+size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
+
+
 /* *************************************
 *  Pre-defined compression levels
 ***************************************/

programs/fuzzer.c

@@ -227,6 +227,7 @@ static int basicUnitTests(U32 seed, double compressibility)
                                            compressedBuffer, cSize,
                                            CNBuffer, dictSize);
         if (ZSTD_isError(result)) goto _output_error;
+        if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
         ZSTD_freeCCtx(ctxOrig);   /* if ctxOrig is read, will produce segfault */
         DISPLAYLEVEL(4, "OK \n");
 
@@ -249,6 +250,7 @@ static int basicUnitTests(U32 seed, double compressibility)
                                            compressedBuffer, cSize,
                                            CNBuffer, dictSize);
         if (ZSTD_isError(result)) goto _output_error;
+        if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
         ZSTD_freeDCtx(dctx);
         DISPLAYLEVEL(4, "OK \n");
     }
@@ -266,6 +268,32 @@ static int basicUnitTests(U32 seed, double compressibility)
     if (!ZSTD_isError(result)) goto _output_error;
     DISPLAYLEVEL(4, "OK \n");
 
+    /* block API tests */
+    {
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+        const size_t blockSize = 100 KB;
+
+        /* basic block compression */
+        DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
+        result = ZSTD_compressBegin(cctx, 5);
+        if (ZSTD_isError(result)) goto _output_error;
+        cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
+        if (ZSTD_isError(cSize)) goto _output_error;
+        DISPLAYLEVEL(4, "OK \n");
+
+        DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
+        result = ZSTD_resetDCtx(dctx);
+        if (ZSTD_isError(result)) goto _output_error;
+        result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
+        if (ZSTD_isError(result)) goto _output_error;
+        if (result != blockSize) goto _output_error;
+        DISPLAYLEVEL(4, "OK \n");
+
+        ZSTD_freeCCtx(cctx);
+        ZSTD_freeDCtx(dctx);
+    }
+
     /* long rle test */
     {
         size_t sampleSize = 0;