diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c index b7937f1d8..844c50405 100644 --- a/lib/zstd_buffered.c +++ b/lib/zstd_buffered.c @@ -204,11 +204,12 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, void* cDst; size_t cSize; size_t iSize = zbc->inBuffPos - zbc->inToCompress; - if ((size_t)(oend-op) > ZSTD_compressBound(iSize)) + size_t oSize = oend-op; + if (oSize >= ZSTD_compressBound(iSize)) cDst = op; /* compress directly into output buffer (avoid flush stage) */ else - cDst = zbc->outBuff; - cSize = ZSTD_compressContinue(zbc->zc, cDst, oend-op, zbc->inBuff + zbc->inToCompress, iSize); + cDst = zbc->outBuff, oSize = zbc->outBuffSize; + cSize = ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize); if (ZSTD_isError(cSize)) return cSize; /* prepare next block */ zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize; diff --git a/programs/Makefile b/programs/Makefile index 3259de07d..65ba9e064 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -171,60 +171,60 @@ test-zstd: zstd datagen @diff tmp1 tmp2 ./datagen | ./zstd -6 -v | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - @./datagen -g257MB | md5sum > tmp1 - ./datagen -g257MB | ./zstd -v | ./zstd -d | md5sum > tmp2 + @./datagen -g270000000 | md5sum > tmp1 + ./datagen -g270000000 | ./zstd -v | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g257MB | ./zstd -v2 | ./zstd -d | md5sum > tmp2 + ./datagen -g270000000 | ./zstd -v2 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g257MB | ./zstd -v3 | ./zstd -d | md5sum > tmp2 + ./datagen -g270000000 | ./zstd -v3 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - @./datagen -g129MB -P60| md5sum > tmp1 - ./datagen -g129MB -P60 | ./zstd -v4 | ./zstd -d | md5sum > tmp2 + @./datagen -g140000000 -P60| md5sum > tmp1 + ./datagen -g140000000 -P60 | ./zstd -v4 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g129MB -P60 | ./zstd -v5 | ./zstd -d | md5sum > tmp2 + ./datagen -g140000000 -P60 | ./zstd -v5 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g129MB -P60 | ./zstd -v6 | ./zstd -d | md5sum > tmp2 + ./datagen -g140000000 -P60 | ./zstd -v6 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - @./datagen -g65MB -P70 | md5sum > tmp1 - ./datagen -g65MB -P70 | ./zstd -v7 | ./zstd -d | md5sum > tmp2 + @./datagen -g70000000 -P70 | md5sum > tmp1 + ./datagen -g70000000 -P70 | ./zstd -v7 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g65MB -P70 | ./zstd -v8 | ./zstd -d | md5sum > tmp2 + ./datagen -g70000000 -P70 | ./zstd -v8 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g65MB -P70 | ./zstd -v9 | ./zstd -d | md5sum > tmp2 + ./datagen -g70000000 -P70 | ./zstd -v9 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - @./datagen -g33MB -P75 | md5sum > tmp1 - ./datagen -g33MB -P75 | ./zstd -v10 | ./zstd -d | md5sum > tmp2 + @./datagen -g35000000 -P75 | md5sum > tmp1 + ./datagen -g35000000 -P75 | ./zstd -v10 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g33MB -P75 | ./zstd -v11 | ./zstd -d | md5sum > tmp2 + ./datagen -g35000000 -P75 | ./zstd -v11 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g33MB -P75 | ./zstd -v12 | ./zstd -d | md5sum > tmp2 + ./datagen -g35000000 -P75 | ./zstd -v12 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - @./datagen -g17MB -P80 | md5sum > tmp1 - ./datagen -g17MB -P80 | ./zstd -v13 | ./zstd -d | md5sum > tmp2 + @./datagen -g18000000 -P80 | md5sum > tmp1 + ./datagen -g18000000 -P80 | ./zstd -v13 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g17MB -P80 | ./zstd -v14 | ./zstd -d | md5sum > tmp2 + ./datagen -g18000000 -P80 | ./zstd -v14 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g17MB -P80 | ./zstd -v15 | ./zstd -d | md5sum > tmp2 + ./datagen -g18000000 -P80 | ./zstd -v15 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g17MB -P80 | ./zstd -v16 | ./zstd -d | md5sum > tmp2 + ./datagen -g18000000 -P80 | ./zstd -v16 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g17MB -P80 | ./zstd -v17 | ./zstd -d | md5sum > tmp2 + ./datagen -g18000000 -P80 | ./zstd -v17 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - @./datagen -g49MB -P93 | md5sum > tmp1 - ./datagen -g49MB -P93 | ./zstd -v18 | ./zstd -d | md5sum > tmp2 + @./datagen -g50000000 -P94 | md5sum > tmp1 + ./datagen -g50000000 -P94 | ./zstd -v18 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g49MB -P93 | ./zstd -v19 | ./zstd -d | md5sum > tmp2 + ./datagen -g50000000 -P94 | ./zstd -v19 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - @./datagen -g97MB -P99 | md5sum > tmp1 - ./datagen -g97MB -P99 | ./zstd -v20 | ./zstd -d | md5sum > tmp2 + @./datagen -g99000000 -P99 | md5sum > tmp1 + ./datagen -g99000000 -P99 | ./zstd -v20 | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g6GB -P99 | md5sum > tmp1 - ./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d | md5sum > tmp2 + ./datagen -g6000000000 -P99| md5sum > tmp1 + ./datagen -g6000000000 -P99| ./zstd -vq | ./zstd -d | md5sum > tmp2 @diff tmp1 tmp2 test-zstd32: zstd32 datagen - ./datagen | ./zstd32 -v | ./zstd32 -d > $(VOID) - ./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID) + ./datagen | ./zstd32 -v | ./zstd32 -d > $(VOID) + ./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID) ./datagen -g6GB -P99 | ./zstd32 -vq | ./zstd32 -d > $(VOID) test-fullbench: fullbench datagen diff --git a/programs/fileio.c b/programs/fileio.c index 267ddb562..7e5f80da6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -337,7 +337,6 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput, size_t inStart=0; size_t toRead = ZBUFF_decompressContinue(dctx, outBuff, &decodedSize, inBuff+inStart, &inSize); if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead)); - if (toRead==0) break; /* end of Frame */ readSize -= inSize; inStart += inSize; @@ -347,6 +346,7 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput, frameSize += decodedSize; DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); + if (toRead == 0) break; if (readSize) continue; /* still some data left within inBuff */ /* Fill input buffer */ diff --git a/programs/zbufftest.c b/programs/zbufftest.c index f7d2f9833..f1fb5e387 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -324,22 +324,29 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); - - XXH64_update(&crc64, srcBuffer+sampleStart, sampleSize); - memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, sampleSize); - readSize = sampleSize; - genSize = cBufferSize - cSize; + + /* random size output buffer */ + sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; + sampleSize = (size_t)1 << sampleSizeLog; + sampleSize += FUZ_rand(&lseed) & (sampleSize-1); + genSize = MIN (cBufferSize - cSize, sampleSize); + errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); - CHECK (readSize != sampleSize, "compression test condition not respected : input should be fully consumed") + + XXH64_update(&crc64, srcBuffer+sampleStart, readSize); + memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize); cSize += genSize; - totalTestSize += sampleSize; + totalTestSize += readSize; if ((FUZ_rand(&lseed) & 15) == 0) { /* add a few random flushes operations, to mess around */ - genSize = cBufferSize - cSize; + sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; + sampleSize = (size_t)1 << sampleSizeLog; + sampleSize += FUZ_rand(&lseed) & (sampleSize-1); + genSize = MIN (cBufferSize - cSize, sampleSize); errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize); CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode)); cSize += genSize;