changed streaming decoder behavior : now, when all compressed frame is consumed, it means decompression is completed, with regenerated data fully flushed.

lib/decompress/zstd_decompress.c

@@ -1286,6 +1286,7 @@ struct ZSTD_DStream_s {
     void* legacyContext;
     U32 previousLegacyVersion;
     U32 legacyVersion;
+    U32 hostageByte;
 };   /* typedef'd to ZSTD_DStream within "zstd.h" */
 
 
@@ -1349,6 +1350,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
         zds->dictSize = dictSize;
     }
     zds->legacyVersion = 0;
+    zds->hostageByte = 0;
     return ZSTD_frameHeaderSize_prefix;
 }
 
@@ -1371,11 +1373,11 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
 
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
 {
-    return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize;
+    return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize + zds->dictSize;
 }
 
 
-/* *** Decompression *** */
+/* *****   Decompression   ***** */
 
 MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
@@ -1445,7 +1447,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
             if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
 
-            /* Frame header instruct buffer sizes */
+            /* Adapt buffer sizes to frame header instructions */
             {   size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
                 size_t const neededOutSize = zds->fParams.windowSize + blockSize;
                 zds->blockSize = blockSize;
@@ -1522,7 +1524,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                         zds->outStart = zds->outEnd = 0;
                     break;
                 }
-                /* cannot flush everything */
+                /* cannot complete flush */
                 someMoreWork = 0;
                 break;
             }
@@ -1533,8 +1535,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     input->pos += (size_t)(ip-istart);
     output->pos += (size_t)(op-ostart);
     {   size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
-        if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart);   /* return 0 only if fully flushed too */
+        if (!nextSrcSizeHint) {   /* frame fully decoded */
-        nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);
+            if (zds->outEnd == zds->outStart) {  /* output fully flushed */
+                if (zds->hostageByte) {
+                    if (input->pos >= input->size) { zds->stage = zdss_read; return 1; }  /* can't release hostage (not present) */
+                    input->pos++;  /* release hostage */
+                }
+                return 0;
+            }
+            if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
+                input->pos--;   /* note : pos > 0, otherwise, impossible to finish reading last block */
+                zds->hostageByte=1;
+            }
+            return 1;
+        }
+        nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);   /* preload header of next block */
         if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC);   /* should never happen */
         nextSrcSizeHint -= zds->inPos;   /* already loaded*/
         return nextSrcSizeHint;

lib/zstd.h

@@ -252,15 +252,13 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 *
 *  Use ZSTD_decompressStream() repetitively to consume your input.
 *  The function will update both `pos` fields.
-*  If `input.pos < input.size`, some input is not consumed.
+*  If `input.pos < input.size`, some input has not been consumed.
 *  It's up to the caller to present again remaining data.
-*  If `output.pos == output.size`, there is probably some more data to flush, still stored inside internal buffers.
+*  If `output.pos < output.size`, decoder has flushed everything it could.
 *  @return : 0 when a frame is completely decoded and fully flushed,
 *            an error code, which can be tested using ZSTD_isError(),
-*            any value > 0, which means there is still some work to do to complete the frame.
+*            any other value > 0, which means there is still some work to do to complete the frame.
-*            In general, the return value is a suggested next input size (merely a hint, to help latency).
+*            The return value is a suggested next input size (just an hint, to help latency).
-*            1 is a special value, which means either "there is still some data to flush", or "need 1 more byte as input".
-*            In which case, start by flushing. When flush is completed, if return value is still `1`, it means "need 1 more byte".
 * *******************************************************************************/
 
 typedef struct ZSTD_DStream_s ZSTD_DStream;

tests/playTests.sh

@@ -45,7 +45,9 @@ file $ZSTD
 $ECHO "\n**** simple tests **** "
 
 ./datagen > tmp
+$ECHO "test : basic compression "
 $ZSTD -f tmp                      # trivial compression case, creates tmp.zst
+$ECHO "test : basic decompression"
 $ZSTD -df tmp.zst                 # trivial decompression case (overwrites tmp)
 $ECHO "test : too large compression level (must fail)"
 $ZSTD -99 -f tmp  # too large compression level, automatic sized down

tests/zstreamtest.c

@@ -58,7 +58,7 @@ static U32 g_displayLevel = 2;
             if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
             { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
             if (g_displayLevel>=4) fflush(stdout); } }
-static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
 static clock_t g_displayClock = 0;
 
 static clock_t g_clockTime = 0;
@@ -118,8 +118,7 @@ static void freeFunction(void* opaque, void* address)
 
 static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
 {
-    int testResult = 0;
+    size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
-    size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
     void* CNBuffer = malloc(CNBufferSize);
     size_t const skippableFrameSize = 11;
     size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
@@ -127,6 +126,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     size_t const decodedBufferSize = CNBufferSize;
     void* decodedBuffer = malloc(decodedBufferSize);
     size_t cSize;
+    int testResult = 0;
     U32 testNb=0;
     ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
     ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
@@ -437,7 +437,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         {   U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
             U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
             maxTestSize = FUZ_rLogLength(&lseed, testLog);
-            dictSize  = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
+            dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
             /* random dictionary selection */
             {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                 dict = srcBuffer + dictStart;
@@ -446,7 +446,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
                 params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
                 params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
                 {   size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0);
-                    CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError));
+                    CHECK (ZSTD_isError(initError),"ZSTD_initCStream_advanced error : %s", ZSTD_getErrorName(initError));
         }   }   }
 
         /* multi-segments compression test */