krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-31 08:37:43 +02:00
Code Issues Releases 1 Activity

fixed ZSTD_decompressBlock() using multiple blocks

Browse Source
This commit is contained in:
Yann Collet
2016-06-27 01:31:35 +02:00
parent 63b5e7a2ea
commit d4f4e58ee1
4 changed files with 37 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
lib/compress/zstd_compress.c
View File

@@ -1063,7 +1063,7 @@ static size_t ZSTD_count_2segments(const BYTE* ip, const BYTE* match, const BYTE
***************************************/ ***************************************/
static const U32 prime3bytes = 506832829U; static const U32 prime3bytes = 506832829U;
static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; } static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; }
static size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */
static const U32 prime4bytes = 2654435761U; static const U32 prime4bytes = 2654435761U;
static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
@@ -2068,18 +2068,27 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src,
} }
/* The optimal parser */ /* The optimal parser */
#include "zstd_opt.h" #include "zstd_opt.h"
static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{ {
#ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_generic(ctx, src, srcSize); ZSTD_compressBlock_opt_generic(ctx, src, srcSize);
#else
(void)ctx; (void)src; (void)srcSize;
return;
#endif
} }
static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{ {
#ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize); ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize);
#else
(void)ctx; (void)src; (void)srcSize;
return;
#endif
} }

7
lib/compress/zstd_opt.h
View File

@@ -34,6 +34,10 @@
/* Note : this file is intended to be included within zstd_compress.c */ /* Note : this file is intended to be included within zstd_compress.c */
#ifndef ZSTD_OPT_H_91842398743
#define ZSTD_OPT_H_91842398743
#define ZSTD_FREQ_DIV 5 #define ZSTD_FREQ_DIV 5
/*-************************************* /*-*************************************
@@ -232,7 +236,6 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
/* Update hashTable3 up to ip (excluded) /* Update hashTable3 up to ip (excluded)
Assumption : always within prefix (ie. not within extDict) */ Assumption : always within prefix (ie. not within extDict) */
FORCE_INLINE FORCE_INLINE
@@ -1039,3 +1042,5 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
#endif /* ZSTD_OPT_H_91842398743 */

5
lib/decompress/zstd_decompress.c
View File

@@ -930,8 +930,11 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
size_t dSize;
ZSTD_checkContinuity(dctx, dst); ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
dctx->previousDstEnd = (char*)dst + dSize;
return dSize;
} }

27
programs/fuzzer.c
View File

@@ -40,7 +40,7 @@
#include <sys/timeb.h> /* timeb */ #include <sys/timeb.h> /* timeb */
#include <string.h> /* strcmp */ #include <string.h> /* strcmp */
#include <time.h> /* clock_t */ #include <time.h> /* clock_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */ #include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */
#include "zdict.h" /* ZDICT_trainFromBuffer */ #include "zdict.h" /* ZDICT_trainFromBuffer */
#include "datagen.h" /* RDG_genBuffer */ #include "datagen.h" /* RDG_genBuffer */
@@ -109,9 +109,9 @@ static unsigned FUZ_highbit32(U32 v32)
} }
#define CHECKTEST(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error #define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
#define CHECK(fn) { CHECKTEST(err, fn); } #define CHECK(fn) { CHECK_V(err, fn); }
#define CHECKPLUS(var, fn, more) { CHECKTEST(var, fn); more; } #define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; }
static int basicUnitTests(U32 seed, double compressibility) static int basicUnitTests(U32 seed, double compressibility)
{ {
size_t const CNBuffSize = 5 MB; size_t const CNBuffSize = 5 MB;
@@ -320,6 +320,7 @@ static int basicUnitTests(U32 seed, double compressibility)
ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx();
static const size_t blockSize = 100 KB; static const size_t blockSize = 100 KB;
static const size_t dictSize = 16 KB; static const size_t dictSize = 16 KB;
size_t cSize2;
/* basic block compression */ /* basic block compression */
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++); DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
@@ -330,7 +331,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++); DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
CHECK( ZSTD_decompressBegin(dctx) ); CHECK( ZSTD_decompressBegin(dctx) );
{ CHECKTEST(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); { CHECK_V(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
if (r != blockSize) goto _output_error; } if (r != blockSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
@@ -339,11 +340,15 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) ); CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize); cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
if (ZSTD_isError(cSize)) goto _output_error; if (ZSTD_isError(cSize)) goto _output_error;
cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize);
if (ZSTD_isError(cSize2)) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++); DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
{ CHECKTEST( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); { CHECK_V( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
if (r != blockSize) goto _output_error; }
{ CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+blockSize, CNBuffSize, (char*)compressedBuffer+cSize, cSize2) );
if (r != blockSize) goto _output_error; } if (r != blockSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
@@ -361,7 +366,7 @@ static int basicUnitTests(U32 seed, double compressibility)
sampleSize += 96 KB; sampleSize += 96 KB;
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
if (ZSTD_isError(cSize)) goto _output_error; if (ZSTD_isError(cSize)) goto _output_error;
{ CHECKTEST(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); { CHECK_V(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
if (regenSize!=sampleSize) goto _output_error; } if (regenSize!=sampleSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
} }
@@ -370,12 +375,12 @@ static int basicUnitTests(U32 seed, double compressibility)
#define ZEROESLENGTH 100 #define ZEROESLENGTH 100
DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
memset(CNBuffer, 0, ZEROESLENGTH); memset(CNBuffer, 0, ZEROESLENGTH);
{ CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) ); { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
cSize = r; } cSize = r; }
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
{ CHECKTEST(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) ); { CHECK_V(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
if (r != ZEROESLENGTH) goto _output_error; } if (r != ZEROESLENGTH) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
@@ -403,13 +408,13 @@ static int basicUnitTests(U32 seed, double compressibility)
((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
} }} } }}
DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++); DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++);
{ CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
CNBuffer, _3BYTESTESTLENGTH, 19) ); CNBuffer, _3BYTESTESTLENGTH, 19) );
cSize = r; } cSize = r; }
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++); DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
{ CHECKTEST(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) ); { CHECK_V(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
if (r != _3BYTESTESTLENGTH) goto _output_error; } if (r != _3BYTESTESTLENGTH) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");