krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-06 16:56:49 +02:00
Code Issues Releases 1 Activity

fixed : init*_advanced() followed by reset() with different pledgedSrcSiz

Browse Source
This commit is contained in:
Yann Collet 2016-09-26 16:41:05 +02:00
parent 2f2639438a
commit cf409a7e2a
3 changed files with 25 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/compress/zstd_compress.c
View File

@ -142,21 +142,8 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
} }
/** ZSTD_checkCParams_advanced() :
temporary work-around, while the compressor compatibility remains limited regarding windowLog < 18 */
size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSize)
{
if (srcSize > (1ULL << ZSTD_WINDOWLOG_MIN)) return ZSTD_checkCParams(cParams);
if (cParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) return ERROR(compressionParameter_unsupported);
if (srcSize <= (1ULL << cParams.windowLog)) cParams.windowLog = ZSTD_WINDOWLOG_MIN; /* fake value - temporary work around */
if (srcSize <= (1ULL << cParams.chainLog)) cParams.chainLog = ZSTD_CHAINLOG_MIN; /* fake value - temporary work around */
if ((srcSize <= (1ULL << cParams.hashLog)) & ((U32)cParams.strategy < (U32)ZSTD_btlazy2)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */
return ZSTD_checkCParams(cParams);
}
/** ZSTD_adjustCParams() : /** ZSTD_adjustCParams() :
optimize cPar for a given input (`srcSize` and `dictSize`). optimize `cPar` for a given input (`srcSize` and `dictSize`).
mostly downsizing to reduce memory consumption and initialization. mostly downsizing to reduce memory consumption and initialization.
Both `srcSize` and `dictSize` are optional (use 0 if unknown), Both `srcSize` and `dictSize` are optional (use 0 if unknown),
but if both are 0, no optimization can be done. but if both are 0, no optimization can be done.
@ -169,7 +156,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0; { U32 const minSrcSize = (srcSize==0) ? 500 : 0;
U64 const rSize = srcSize + dictSize + minSrcSize; U64 const rSize = srcSize + dictSize + minSrcSize;
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) { if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
U32 const srcLog = MAX(4, ZSTD_highbit32((U32)(rSize)-1) + 1); U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1);
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
} } } }
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
@ -178,7 +165,6 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
if ((cPar.hashLog < ZSTD_HASHLOG_MIN) & ((U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
return cPar; return cPar;
} }
@ -2556,7 +2542,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
ZSTD_parameters params, unsigned long long pledgedSrcSize) ZSTD_parameters params, unsigned long long pledgedSrcSize)
{ {
/* compression parameters verification and optimization */ /* compression parameters verification and optimization */
CHECK_F(ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize)); CHECK_F(ZSTD_checkCParams(params.cParams));
return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize);
} }
@ -2644,7 +2630,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
const void* dict,size_t dictSize, const void* dict,size_t dictSize,
ZSTD_parameters params) ZSTD_parameters params)
{ {
CHECK_F(ZSTD_checkCParams_advanced(params.cParams, srcSize)); CHECK_F(ZSTD_checkCParams(params.cParams));
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
} }
@ -2851,7 +2837,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
if (zcs->inBuffSize < neededInBuffSize) { if (zcs->inBuffSize < neededInBuffSize) {
zcs->inBuffSize = neededInBuffSize; zcs->inBuffSize = neededInBuffSize;
ZSTD_free(zcs->inBuff, zcs->customMem); /* should not be necessary */ ZSTD_free(zcs->inBuff, zcs->customMem);
zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem); zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem);
if (zcs->inBuff == NULL) return ERROR(memory_allocation); if (zcs->inBuff == NULL) return ERROR(memory_allocation);
} }
@ -2859,7 +2845,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
} }
if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) { if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) {
zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1;
ZSTD_free(zcs->outBuff, zcs->customMem); /* should not be necessary */ ZSTD_free(zcs->outBuff, zcs->customMem);
zcs->outBuff = (char*) ZSTD_malloc(zcs->outBuffSize, zcs->customMem); zcs->outBuff = (char*) ZSTD_malloc(zcs->outBuffSize, zcs->customMem);
if (zcs->outBuff == NULL) return ERROR(memory_allocation); if (zcs->outBuff == NULL) return ERROR(memory_allocation);
} }

8
lib/zstd.h
View File

@ -290,11 +290,11 @@ ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* outp
#define ZSTD_WINDOWLOG_MAX_32 25 #define ZSTD_WINDOWLOG_MAX_32 25
#define ZSTD_WINDOWLOG_MAX_64 27 #define ZSTD_WINDOWLOG_MAX_64 27
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) #define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
#define ZSTD_WINDOWLOG_MIN 18 #define ZSTD_WINDOWLOG_MIN 10
#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CHAINLOG_MIN 4
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX #define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
#define ZSTD_HASHLOG_MIN 12 #define ZSTD_HASHLOG_MIN 6
#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
#define ZSTD_HASHLOG3_MAX 17 #define ZSTD_HASHLOG3_MAX 17
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
#define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLOG_MIN 1

22
tests/zstreamtest.c
View File

@ -374,7 +374,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
ZSTD_DStream* const zd_noise = ZSTD_createDStream(); ZSTD_DStream* const zd_noise = ZSTD_createDStream();
clock_t const startClock = clock(); clock_t const startClock = clock();
const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */ const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize=0, maxTestSize=0; size_t dictSize = 0;
U32 oldTestLog = 0;
/* allocations */ /* allocations */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@ -407,6 +408,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
XXH64_state_t xxhState; XXH64_state_t xxhState;
U64 crcOrig; U64 crcOrig;
U32 resetAllowed = 1; U32 resetAllowed = 1;
size_t maxTestSize;
/* init */ /* init */
DISPLAYUPDATE(2, "\r%6u", testNb); DISPLAYUPDATE(2, "\r%6u", testNb);
@ -435,23 +437,29 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
} }
/* compression init */ /* compression init */
if (maxTestSize /* at least one test happened */ && resetAllowed && (FUZ_rand(&lseed)&1)) { if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; && oldTestLog /* at least one test happened */ && resetAllowed) {
ZSTD_resetCStream(zc, pledgedSrcSize); maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
size_t const resetError = ZSTD_resetCStream(zc, pledgedSrcSize);
CHECK(ZSTD_isError(resetError), "ZSTD_resetCStream error : %s", ZSTD_getErrorName(resetError));
}
} else { } else {
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog); maxTestSize = FUZ_rLogLength(&lseed, testLog);
oldTestLog = testLog;
/* random dictionary selection */ /* random dictionary selection */
dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart; dict = srcBuffer + dictStart;
} }
{ ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize); { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; { size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize);
size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize);
CHECK (ZSTD_isError(initError),"ZSTD_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); CHECK (ZSTD_isError(initError),"ZSTD_initCStream_advanced error : %s", ZSTD_getErrorName(initError));
} } } } } }