New streaming API behavior, to solve issue 19 (https://github.com/Cyan4973/zstd/issues/19)

lib/zstd.c

@@ -141,7 +141,7 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51C;   /* Initial (limited) frame for
 
 #define KB *(1 <<10)
 #define MB *(1 <<20)
-#define GB *(1U<<20)
+#define GB *(1U<<30)
 
 #define BLOCKSIZE (128 KB)                 /* define, for static allocation */
 static const U32 g_maxDistance = 512 KB;
@@ -1708,24 +1708,24 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
 }
 
 
-/******************************
+/*******************************
 *  Streaming Decompression API
-******************************/
+*******************************/
 
 typedef struct
 {
     U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)];
     size_t expected;
     blockType_t bType;
-    U32 started;
+    U32 phase;
 } dctx_t;
 
 
 ZSTD_dctx_t ZSTD_createDCtx(void)
 {
     dctx_t* dctx = (dctx_t*)malloc(sizeof(dctx_t));
-    dctx->expected = 4 + ZSTD_blockHeaderSize;   // Frame Header + Block Header
+    dctx->expected = ZSTD_frameHeaderSize;
-    dctx->started = 0;
+    dctx->phase = 0;
     return (ZSTD_dctx_t)dctx;
 }
 
@@ -1736,7 +1736,7 @@ size_t ZSTD_freeDCtx(ZSTD_dctx_t dctx)
 }
 
 
-size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx)
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_dctx_t dctx)
 {
     return ((dctx_t*)dctx)->expected;
 }
@@ -1744,63 +1744,67 @@ size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx)
 size_t ZSTD_decompressContinue(ZSTD_dctx_t dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
 {
     dctx_t* ctx = (dctx_t*)dctx;
-    size_t cSize = srcSize - ZSTD_blockHeaderSize;
-    size_t rSize;
 
-    // Sanity check
+    /* Sanity check */
     if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_wrongSrcSize;
 
-    // Decompress
+    /* Decompress : frame header */
-    if (!ctx->started)
+    if (ctx->phase == 0)
     {
-        // Just check correct magic header
+        /* Check frame magic header */
         U32 magicNumber = ZSTD_readBE32(src);
         if (magicNumber != ZSTD_magicNumber) return (size_t)-ZSTD_ERROR_wrongMagicNumber;
-        rSize = 0;
+        ctx->phase = 1;
+        ctx->expected = ZSTD_blockHeaderSize;
+        return 0;
     }
-    else
+
+    /* Decompress : block header */
+    if (ctx->phase == 1)
     {
+        blockProperties_t bp;
+        size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
+        if (ZSTD_isError(blockSize)) return blockSize;
+        if (bp.blockType == bt_end)
+        {
+            ctx->expected = 0;
+            ctx->phase = 0;
+        }
+        else
+        {
+            ctx->expected = blockSize;
+            ctx->bType = bp.blockType;
+            ctx->phase = 2;
+        }
+
+        return 0;
+    }
+
+    /* Decompress : block content */
+    {
+        size_t rSize;
         switch(ctx->bType)
         {
         case bt_compressed:
-            rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, cSize);
+            rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize);
             break;
         case bt_raw :
-            rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, cSize);
+            rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, srcSize);
             break;
         case bt_rle :
             return (size_t)-ZSTD_ERROR_GENERIC;   /* not yet handled */
             break;
-        case bt_end :
+        case bt_end :   /* should never happen (filtered at phase 1) */
             rSize = 0;
             break;
         default:
             return (size_t)-ZSTD_ERROR_GENERIC;
         }
+        ctx->phase = 1;
+        ctx->expected = ZSTD_blockHeaderSize;
+        return rSize;
     }
 
-    // Prepare next block
-    {
-        const BYTE* header = (const BYTE*)src;
-        blockProperties_t bp;
-        size_t blockSize;
-        header += cSize;
-        blockSize = ZSTD_getcBlockSize(header, ZSTD_blockHeaderSize, &bp);
-        if (ZSTD_isError(blockSize)) return blockSize;
-        if (bp.blockType == bt_end)
-        {
-            ctx->expected = 0;
-            ctx->started = 0;
-        }
-        else
-        {
-            ctx->expected = blockSize + ZSTD_blockHeaderSize;
-            ctx->bType = bp.blockType;
-            ctx->started = 1;
-        }
-    }
-
-    return rSize;
 }
 
 

lib/zstd.h

@@ -47,7 +47,7 @@ extern "C" {
 **************************************/
 #define ZSTD_VERSION_MAJOR    0    /* for breaking interface changes  */
 #define ZSTD_VERSION_MINOR    0    /* for new (non-breaking) interface capabilities */
-#define ZSTD_VERSION_RELEASE  1    /* for tweaks, bug-fixes, or development */
+#define ZSTD_VERSION_RELEASE  2    /* for tweaks, bug-fixes, or development */
 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 unsigned ZSTD_versionNumber (void);
 

lib/zstd_static.h

@@ -57,7 +57,7 @@ typedef void* ZSTD_dctx_t;
 ZSTD_dctx_t ZSTD_createDCtx(void);
 size_t      ZSTD_freeDCtx(ZSTD_dctx_t dctx);
 
-size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx);
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_dctx_t dctx);
 size_t ZSTD_decompressContinue(ZSTD_dctx_t dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
 
 
@@ -77,4 +77,4 @@ typedef enum { ZSTD_LIST_ERRORS(ZSTD_GENERATE_ENUM) } ZSTD_errorCodes;   /* expo
 
 #if defined (__cplusplus)
 }
-#endif
+#endif

programs/fileio.c

@@ -327,10 +327,10 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
     /* Init */
     FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
     dctx = ZSTD_createDCtx();
-    toRead = ZSTD_getNextcBlockSize(dctx);
-    if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
 
     /* check header */
+    toRead = ZSTD_nextSrcSizeToDecompress(dctx);
+    if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
     sizeCheck = fread(header, (size_t)1, toRead, finput);
     if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
     sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead);   // Decode frame header
@@ -348,7 +348,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
     if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
 
     /* Main decompression Loop */
-    toRead = ZSTD_getNextcBlockSize(dctx);
+    toRead = ZSTD_nextSrcSizeToDecompress(dctx);
     while (toRead)
     {
         size_t readSize, decodedSize;
@@ -361,16 +361,19 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
         /* Decode block */
         decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize);
 
-        /* Write block */
+        if (decodedSize)   /* not a header */
-        sizeCheck = fwrite(op, 1, decodedSize, foutput);
+        {
-        if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file");
+            /* Write block */
-        filesize += decodedSize;
+            sizeCheck = fwrite(op, 1, decodedSize, foutput);
+            if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file");
+            filesize += decodedSize;
+            op += decodedSize;
+            if (op==oend) op = outBuff;
+            DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(filesize>>20) );
+        }
 
         /* prepare for next Block */
-        op += decodedSize;
+        toRead = ZSTD_nextSrcSizeToDecompress(dctx);
-        if (op==oend) op = outBuff;
-        toRead = ZSTD_getNextcBlockSize(dctx);
-        DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(filesize>>20) );
     }
 
     DISPLAYLEVEL(2, "\r%79s\r", "");