krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-07 01:10:04 +02:00
Code Issues Releases 1 Activity

Fixed 32-bits windowLog limitation

Browse Source 
Fixed some macro errors in zbufftest
This commit is contained in:
Yann Collet 2016-04-04 04:22:53 +02:00
parent 9bf7016627
commit 1535414927
4 changed files with 32 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lib/zbuff.c
View File

@ -26,14 +26,9 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
- zstd homepage : http://www.zstd.net/
*/
/* The objects defined into this file should be considered experimental.
* They are not labelled stable, as their prototype may change in the future.
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
*/
/* *************************************
* Dependencies
@ -281,7 +276,7 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
BYTE* op = ostart;
size_t outSize = *dstCapacityPtr;
size_t epilogueSize, remaining;
ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
op += outSize;
epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */
zbc->outBuffContentSize += epilogueSize;
@ -289,7 +284,7 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
zbc->stage = ZBUFFcs_flush;
remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */
op += outSize;
if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */
if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */
*dstCapacityPtr = op-ostart; /* tells how many bytes were written */
return remaining;
}
@ -466,6 +461,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
zbc->stage = ZBUFFds_flush;
// break; /* ZBUFFds_flush follows */
} }
case ZBUFFds_flush:
{ size_t const toFlushSize = zbc->outEnd - zbc->outStart;
size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);

12
lib/zstd_compress.c
View File

@ -141,8 +141,7 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface *
@return : 0, or an error code if one value is beyond authorized range */
size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
{
{ U32 const windowLog_max = MEM_32bits() ? 25 : ZSTD_WINDOWLOG_MAX; /* 32 bits mode cannot flush > 24 bits */
CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, windowLog_max); }
CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
CLAMPCHECK(cParams.contentLog, ZSTD_CONTENTLOG_MIN, ZSTD_CONTENTLOG_MAX);
CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
@ -2413,7 +2412,6 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
#define ZSTD_MAX_CLEVEL 22
unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
{ /* "default" */
/* W, C, H, S, L, SL, strat */
@ -2526,14 +2524,16 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
* `srcSize` value is optional, select 0 if not known */
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize)
{
ZSTD_compressionParameters cp;
size_t addedSize = srcSize ? 0 : 500;
U64 rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1;
U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1;
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
#if ZSTD_OPT_DEBUG >= 1
tableID=0;
#endif
return ZSTD_defaultCParameters[tableID][compressionLevel];
cp = ZSTD_defaultCParameters[tableID][compressionLevel];
if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX; /* auto-correction, for 32-bits mode */
return cp;
}

2
lib/zstd_static.h
View File

@ -57,7 +57,7 @@ extern "C" {
/*-*************************************
* Types
***************************************/
#define ZSTD_WINDOWLOG_MAX 27
#define ZSTD_WINDOWLOG_MAX (MEM_32bits() ? 25 : 27)
#define ZSTD_WINDOWLOG_MIN 18
#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CONTENTLOG_MIN 4

40
programs/zbufftest.c
View File

@ -134,12 +134,12 @@ static unsigned FUZ_highbit32(U32 v32)
static int basicUnitTests(U32 seed, double compressibility)
{
int testResult = 0;
void* CNBuffer;
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
void* compressedBuffer;
size_t compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
void* decodedBuffer;
size_t decodedBufferSize = CNBufferSize;
void* CNBuffer = malloc(CNBufferSize);
size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
void* compressedBuffer = malloc(compressedBufferSize);
size_t const decodedBufferSize = CNBufferSize;
void* decodedBuffer = malloc(decodedBufferSize);
U32 randState = seed;
size_t result, cSize, readSize, genSize;
U32 testNb=0;
@ -147,9 +147,6 @@ static int basicUnitTests(U32 seed, double compressibility)
ZBUFF_DCtx* zd = ZBUFF_createDCtx();
/* Create compressible test buffer */
CNBuffer = malloc(CNBufferSize);
compressedBuffer = malloc(compressedBufferSize);
decodedBuffer = malloc(decodedBufferSize);
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
DISPLAY("Not enough memory, aborting\n");
goto _output_error;
@ -242,7 +239,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
BYTE* cNoiseBuffer[5];
size_t srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* copyBuffer;
size_t copyBufferSize = srcBufferSize + (1<<maxSampleLog);
size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
BYTE* cBuffer;
size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* dstBuffer;
@ -282,7 +279,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) {
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) {
U32 lseed;
const BYTE* srcBuffer;
const BYTE* dict;
@ -329,8 +326,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel);
}
{ size_t const initError = ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel);
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
} }
/* multi-segments compression test */
XXH64_reset(xxh64, 0);
@ -338,7 +336,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
/* compress random chunk into random size dst buffer */
{ size_t readChunkSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(cBufferSize - cSize, FUZ_randomLength(&lseed, maxSampleLog));
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize);
size_t const compressionError = ZBUFF_compressContinue(zc, cBuffer+cSize, &dstBuffSize, srcBuffer+srcStart, &readChunkSize);
@ -352,7 +351,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
/* random flush operation, to mess around */
if ((FUZ_rand(&lseed) & 15) == 0) {
size_t dstBuffSize = MIN(cBufferSize - cSize, FUZ_randomLength(&lseed, maxSampleLog));
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
size_t const flushError = ZBUFF_compressFlush(zc, cBuffer+cSize, &dstBuffSize);
CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
cSize += dstBuffSize;
@ -370,7 +370,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
ZBUFF_decompressInitDictionary(zd, dict, dictSize);
for (totalCSize = 0, totalGenSize = 0 ; totalCSize < cSize ; ) {
size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, FUZ_randomLength(&lseed, maxSampleLog));
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
size_t const decompressError = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
CHECK (ZBUFF_isError(decompressError), "decompression error : %s", ZBUFF_getErrorName(decompressError));
totalGenSize += dstBuffSize;
@ -389,7 +390,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
/* add some noise */
{ U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
size_t const noiseSize = MIN(cSize/3, FUZ_randomLength(&lseed, maxSampleLog));
size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
@ -401,7 +403,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
totalGenSize = 0;
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) {
size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, FUZ_randomLength(&lseed, maxSampleLog));
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
size_t const decompressError = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
if (ZBUFF_isError(decompressError)) break; /* error correctly detected */
totalGenSize += dstBuffSize;
@ -463,10 +466,9 @@ int main(int argc, const char** argv)
/* Check command line */
for(argNb=1; argNb<argc; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
/* Handle commands. Aggregated commands are allowed */
/* Parsing commands. Aggregated commands are allowed */
if (argument[0]=='-') {
argument++;