From b3060f7a9ea3555c6045606be58dddc86bbb099b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 9 Sep 2016 16:44:16 +0200 Subject: [PATCH] changed streaming decoder behavior : now, when all compressed frame is consumed, it means decompression is completed, with regenerated data fully flushed. --- lib/decompress/zstd_decompress.c | 29 ++++++++++++++++++++++------- lib/zstd.h | 10 ++++------ tests/playTests.sh | 2 ++ tests/zstreamtest.c | 10 +++++----- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 6acb259bd..c6bb5329c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1286,6 +1286,7 @@ struct ZSTD_DStream_s { void* legacyContext; U32 previousLegacyVersion; U32 legacyVersion; + U32 hostageByte; }; /* typedef'd to ZSTD_DStream within "zstd.h" */ @@ -1349,6 +1350,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di zds->dictSize = dictSize; } zds->legacyVersion = 0; + zds->hostageByte = 0; return ZSTD_frameHeaderSize_prefix; } @@ -1371,11 +1373,11 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { - return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize; + return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize + zds->dictSize; } -/* *** Decompression *** */ +/* ***** Decompression ***** */ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { @@ -1445,7 +1447,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported); - /* Frame header instruct buffer sizes */ + /* Adapt buffer sizes to frame header instructions */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); size_t const neededOutSize = zds->fParams.windowSize + blockSize; zds->blockSize = blockSize; @@ -1479,7 +1481,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (ZSTD_isError(decodedSize)) return decodedSize; ip += neededInSize; if (!decodedSize && !isSkipFrame) break; /* this was just a header */ - zds->outEnd = zds->outStart + decodedSize; + zds->outEnd = zds->outStart + decodedSize; zds->stage = zdss_flush; break; } @@ -1522,7 +1524,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->outStart = zds->outEnd = 0; break; } - /* cannot flush everything */ + /* cannot complete flush */ someMoreWork = 0; break; } @@ -1533,8 +1535,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB input->pos += (size_t)(ip-istart); output->pos += (size_t)(op-ostart); { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd); - if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */ - nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { zds->stage = zdss_read; return 1; } /* can't release hostage (not present) */ + input->pos++; /* release hostage */ + } + return 0; + } + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); /* preload header of next block */ if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ nextSrcSizeHint -= zds->inPos; /* already loaded*/ return nextSrcSizeHint; diff --git a/lib/zstd.h b/lib/zstd.h index 10312fe81..5cc40c63c 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -252,15 +252,13 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * * Use ZSTD_decompressStream() repetitively to consume your input. * The function will update both `pos` fields. -* If `input.pos < input.size`, some input is not consumed. +* If `input.pos < input.size`, some input has not been consumed. * It's up to the caller to present again remaining data. -* If `output.pos == output.size`, there is probably some more data to flush, still stored inside internal buffers. +* If `output.pos < output.size`, decoder has flushed everything it could. * @return : 0 when a frame is completely decoded and fully flushed, * an error code, which can be tested using ZSTD_isError(), -* any value > 0, which means there is still some work to do to complete the frame. -* In general, the return value is a suggested next input size (merely a hint, to help latency). -* 1 is a special value, which means either "there is still some data to flush", or "need 1 more byte as input". -* In which case, start by flushing. When flush is completed, if return value is still `1`, it means "need 1 more byte". +* any other value > 0, which means there is still some work to do to complete the frame. +* The return value is a suggested next input size (just an hint, to help latency). * *******************************************************************************/ typedef struct ZSTD_DStream_s ZSTD_DStream; diff --git a/tests/playTests.sh b/tests/playTests.sh index 64b3fd956..21e98bf90 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -45,7 +45,9 @@ file $ZSTD $ECHO "\n**** simple tests **** " ./datagen > tmp +$ECHO "test : basic compression " $ZSTD -f tmp # trivial compression case, creates tmp.zst +$ECHO "test : basic decompression" $ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) $ECHO "test : too large compression level (must fail)" $ZSTD -99 -f tmp # too large compression level, automatic sized down diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 305363877..97fbaa18e 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -58,7 +58,7 @@ static U32 g_displayLevel = 2; if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ if (g_displayLevel>=4) fflush(stdout); } } -static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100; +static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_displayClock = 0; static clock_t g_clockTime = 0; @@ -118,8 +118,7 @@ static void freeFunction(void* opaque, void* address) static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem) { - int testResult = 0; - size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; + size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; void* CNBuffer = malloc(CNBufferSize); size_t const skippableFrameSize = 11; size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); @@ -127,6 +126,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo size_t const decodedBufferSize = CNBufferSize; void* decodedBuffer = malloc(decodedBufferSize); size_t cSize; + int testResult = 0; U32 testNb=0; ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem); ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem); @@ -437,7 +437,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; maxTestSize = FUZ_rLogLength(&lseed, testLog); - dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; + dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; /* random dictionary selection */ { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); dict = srcBuffer + dictStart; @@ -446,7 +446,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; { size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0); - CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError)); + CHECK (ZSTD_isError(initError),"ZSTD_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); } } } /* multi-segments compression test */