krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-07 01:10:04 +02:00
Code Issues Releases 1 Activity

modified buffer management rules

Browse Source
This commit is contained in:
Yann Collet 2016-02-02 14:36:49 +01:00
parent 82d71fe89f
commit 2ce4923238
4 changed files with 63 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
lib/zstd_compress.c
View File

@ -114,6 +114,7 @@ struct ZSTD_CCtx_s
U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more data */
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 loadedDictEnd;
U32 stage;
ZSTD_parameters params;
void* workSpace;
@ -217,6 +218,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2);
zc->hbSize = 0;
zc->stage = 0;
zc->loadedDictEnd = 0;
return 0;
}
@ -906,7 +908,7 @@ static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
}
/* *************************************
/*-*************************************
* Fast Scan
***************************************/
#define FILLHASHSTEP 3
@ -916,7 +918,7 @@ static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls)
const U32 hBits = zc->params.hashLog;
const BYTE* const base = zc->base;
const BYTE* ip = base + zc->nextToUpdate;
const BYTE* const iend = (const BYTE*) end;
const BYTE* const iend = ((const BYTE*)end) - 8;
while(ip <= iend) {
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
@ -1885,47 +1887,41 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit);
if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */
if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */
blockCompressor(zc, src, srcSize);
return ZSTD_compressSequences(zc, dst, maxDstSize, srcSize);
}
static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr,
static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
size_t blockSize = ctxPtr->blockSize;
size_t blockSize = zc->blockSize;
size_t remaining = srcSize;
const BYTE* ip = (const BYTE*)src;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
const U32 maxDist = 1 << ctxPtr->params.windowLog;
const U32 maxDist = 1 << zc->params.windowLog;
while (remaining)
{
while (remaining) {
size_t cSize;
if (maxDstSize < 3 + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (maxDstSize < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize) blockSize = remaining;
if ((U32)(ip+blockSize - (ctxPtr->base + ctxPtr->lowLimit)) > maxDist)
{
/* respect windowLog contract */
ctxPtr->lowLimit = (U32)(ip+blockSize - ctxPtr->base) - maxDist;
if (ctxPtr->dictLimit < ctxPtr->lowLimit) ctxPtr->dictLimit = ctxPtr->lowLimit;
if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { /* enforce maxDist */
zc->lowLimit = (U32)(ip+blockSize - zc->base) - maxDist;
if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit;
}
cSize = ZSTD_compressBlock_internal(ctxPtr, op+3, maxDstSize-3, ip, blockSize);
cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
if (cSize == 0)
{
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
if (cSize == 0) { /* block is not compressible */
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
}
else
{
} else {
op[0] = (BYTE)(cSize>>16);
op[1] = (BYTE)(cSize>>8);
op[2] = (BYTE)cSize;
@ -1951,8 +1947,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
const BYTE* const ip = (const BYTE*) src;
size_t hbSize = 0;
if (frame && (zc->stage==0))
{
if (frame && (zc->stage==0)) {
hbSize = zc->hbSize;
if (dstSize <= hbSize) return ERROR(dstSize_tooSmall);
zc->stage = 1;
@ -1962,8 +1957,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
}
/* Check if blocks follow each other */
if (src != zc->nextSrc)
{
if (src != zc->nextSrc) {
/* not contiguous */
size_t delta = zc->nextSrc - ip;
zc->lowLimit = zc->dictLimit;
@ -1975,8 +1969,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
}
/* preemptive overflow correction */
if (zc->lowLimit > (1<<30))
{
if (zc->lowLimit > (1<<30)) {
U32 btplus = (zc->params.strategy == ZSTD_btlazy2);
U32 contentMask = (1 << (zc->params.contentLog - btplus)) - 1;
U32 newLowLimit = zc->lowLimit & contentMask; /* preserve position % contentSize */
@ -1991,8 +1984,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
}
/* if input and dictionary overlap : reduce dictionary (presumed modified by input) */
if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit))
{
if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit)) {
zc->lowLimit = (U32)(ip + srcSize - zc->dictBase);
if (zc->lowLimit > zc->dictLimit) zc->lowLimit = zc->dictLimit;
}
@ -2034,6 +2026,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
zc->dictBase = zc->base;
zc->base += ip - zc->nextSrc;
zc->nextToUpdate = zc->dictLimit;
zc->loadedDictEnd = (U32)(iend - zc->base);
zc->nextSrc = iend;
if (srcSize <= 8) return 0;
@ -2041,7 +2034,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
switch(zc->params.strategy)
{
case ZSTD_fast:
ZSTD_fillHashTable (zc, iend-8, zc->params.searchLength);
ZSTD_fillHashTable (zc, iend, zc->params.searchLength);
break;
case ZSTD_greedy:
@ -2052,13 +2045,13 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
case ZSTD_btlazy2:
ZSTD_updateTree(zc, iend-8, iend, 1 << zc->params.searchLog, zc->params.searchLength);
zc->nextToUpdate = (U32)(iend - zc->base);
break;
default:
return ERROR(GENERIC); /* strategy doesn't exist; impossible */
}
zc->nextToUpdate = zc->loadedDictEnd;
return 0;
}
@ -2112,8 +2105,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{
if (dict && dictSize)
{
if (dict && (dictSize>4)) {
U32 magic = MEM_readLE32(dict);
size_t eSize;
if (magic != ZSTD_DICT_MAGIC)
@ -2151,7 +2143,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
{
return ZSTD_compressBegin_advanced(zc, dict, dictSize, ZSTD_getParams(compressionLevel, 0));
return ZSTD_compressBegin_advanced(zc, dict, dictSize, ZSTD_getParams(compressionLevel, 128 KB));
}
size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
@ -2235,7 +2227,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
{
return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize+dictSize));
return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize));
}
size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)

3
programs/bench.c
View File

@ -212,6 +212,7 @@ typedef struct
} blockParam_t;
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b))
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel,
@ -289,7 +290,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
while (BMK_GetMilliStart() == milliTime);
milliTime = BMK_GetMilliStart();
while (BMK_GetMilliSpan(milliTime) < TIMELOOP) {
ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, ZSTD_getParams(cLevel, dictBufferSize+largestBlockSize));
ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, ZSTD_getParams(cLevel, MAX(dictBufferSize, largestBlockSize)));
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,

42
programs/datagen.c
View File

@ -23,7 +23,7 @@
- Public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
/*-************************************
* Includes
**************************************/
#include <stdlib.h> /* malloc */
@ -31,7 +31,7 @@
#include <string.h> /* memcpy */
/**************************************
/*-************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
@ -50,7 +50,7 @@
#endif
/**************************************
/*-************************************
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
@ -62,7 +62,7 @@
#endif
/**************************************
/*-************************************
* Constants
**************************************/
#define KB *(1 <<10)
@ -71,7 +71,7 @@
#define PRIME2 2246822519U
/**************************************
/*-************************************
* Local types
**************************************/
#define LTLOG 13
@ -81,7 +81,7 @@ typedef BYTE litDistribTable[LTSIZE];
/*********************************************************
/*-*******************************************************
* Local Functions
*********************************************************/
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
@ -103,14 +103,12 @@ static void RDG_fillLiteralDistrib(litDistribTable lt, double ld)
BYTE firstChar = '(';
BYTE lastChar = '}';
if (ld==0.0)
{
if (ld==0.0) {
character = 0;
firstChar = 0;
lastChar =255;
}
while (i<LTSIZE)
{
while (i<LTSIZE) {
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
U32 end;
if (weight + i > LTSIZE) weight = LTSIZE-i;
@ -140,13 +138,11 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
U32 prevOffset = 1;
/* special case : sparse content */
while (matchProba >= 1.0)
{
while (matchProba >= 1.0) {
size_t size0 = RDG_rand(seed) & 3;
size0 = (size_t)1 << (16 + size0 * 2);
size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/
if (buffSize < pos + size0)
{
if (buffSize < pos + size0) {
memset(buffPtr+pos, 0, buffSize-pos);
return;
}
@ -160,11 +156,9 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1;
/* Generate compressible data */
while (pos < buffSize)
{
while (pos < buffSize) {
/* Select : Literal (char) or Match (within 32K) */
if (RDG_RAND15BITS < matchProba32)
{
if (RDG_RAND15BITS < matchProba32) {
/* Copy (within 32K) */
size_t match;
size_t d;
@ -178,17 +172,14 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
d = pos + length;
if (d > buffSize) d = buffSize;
while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */
}
else
{
} else {
/* Literal (noise) */
size_t d;
size_t length = RDG_RANDLENGTH;
d = pos + length;
if (d > buffSize) d = buffSize;
while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt);
}
}
} }
}
@ -220,8 +211,7 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba,
RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, lt, &seed);
/* Generate compressible data */
while (total < size)
{
while (total < size) {
RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, lt, &seed);
if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total);
total += genBlockSize;
@ -230,6 +220,6 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba,
memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE);
}
// cleanup
/* cleanup */
free(buff);
}

59
programs/fileio.c
View File

@ -193,42 +193,30 @@ static U64 FIO_getFileSize(const char* infilename)
static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
const char* dstFileName, const char* srcFileName)
{
if (!strcmp (srcFileName, stdinmark))
{
if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input\n");
*fileInPtr = stdin;
SET_BINARY_MODE(stdin);
}
else
{
} else {
*fileInPtr = fopen(srcFileName, "rb");
}
if ( *fileInPtr==0 )
{
if ( *fileInPtr==0 ) {
DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", srcFileName);
return 1;
}
if (!strcmp (dstFileName, stdoutmark))
{
if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output\n");
*fileOutPtr = stdout;
SET_BINARY_MODE(stdout);
}
else
{
/* Check if destination file already exists */
if (!g_overwrite)
{
} else {
if (!g_overwrite) { /* Check if destination file already exists */
*fileOutPtr = fopen( dstFileName, "rb" );
if (*fileOutPtr != 0)
{
/* prompt for overwrite authorization */
if (*fileOutPtr != 0) { /* dest file exists, prompt for overwrite authorization */
fclose(*fileOutPtr);
DISPLAY("Warning : %s already exists \n", dstFileName);
if ((g_displayLevel <= 1) || (*fileInPtr == stdin))
{
if ((g_displayLevel <= 1) || (*fileInPtr == stdin)) {
/* No interaction possible */
DISPLAY("Operation aborted : %s already exists \n", dstFileName);
return 1;
@ -236,15 +224,12 @@ static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
DISPLAY("Overwrite ? (y/N) : ");
{
int ch = getchar();
if ((ch!='Y') && (ch!='y'))
{
if ((ch!='Y') && (ch!='y')) {
DISPLAY("No. Operation aborted : %s already exists \n", dstFileName);
return 1;
}
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
}
}
}
} } }
*fileOutPtr = fopen( dstFileName, "wb" );
}
@ -265,15 +250,13 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
U64 fileSize;
*bufferPtr = NULL;
if (fileName == NULL)
return 0;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = fopen(fileName, "rb");
if (fileHandle==0) EXM_THROW(31, "Error opening file %s", fileName);
fileSize = FIO_getFileSize(fileName);
if (fileSize > MAX_DICT_SIZE)
{
if (fileSize > MAX_DICT_SIZE) {
int seekResult;
if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName); /* avoid extreme cases */
DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, MAX_DICT_SIZE);
@ -455,8 +438,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
ress = FIO_createCResources(dictFileName);
/* loop on each file */
for (u=0; u<nbFiles; u++)
{
for (u=0; u<nbFiles; u++) {
size_t ifnSize = strlen(inFileNamesTable[u]);
if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
strcpy(dstFileName, inFileNamesTable[u]);
@ -525,8 +507,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
/* Main decompression Loop */
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1)
{
while (1) {
/* Decode */
size_t sizeCheck;
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
@ -564,8 +545,7 @@ static int FIO_decompressFile_extRess(dRess_t ress,
if (FIO_getFiles(&dstFile, &srcFile, dstFileName, srcFileName)) return 1;
/* for each frame */
for ( ; ; )
{
for ( ; ; ) {
size_t sizeCheck;
/* check magic number -> version */
size_t toRead = 4;
@ -573,8 +553,7 @@ static int FIO_decompressFile_extRess(dRess_t ress,
if (sizeCheck==0) break; /* no more input */
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(MEM_readLE32(ress.srcBuffer)))
{
if (ZSTD_isLegacy(MEM_readLE32(ress.srcBuffer))) {
filesize += FIO_decompressLegacyFrame(dstFile, srcFile, MEM_readLE32(ress.srcBuffer));
continue;
}
@ -624,14 +603,12 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
if (dstFileName==NULL) EXM_THROW(70, "not enough memory for dstFileName");
ress = FIO_createDResources(dictFileName);
for (u=0; u<nbFiles; u++)
{
for (u=0; u<nbFiles; u++) {
const char* srcFileName = srcNamesTable[u];
size_t sfnSize = strlen(srcFileName);
const char* suffixPtr = srcFileName + sfnSize - suffixSize;
if (dfnSize <= sfnSize-suffixSize+1) { free(dstFileName); dfnSize = sfnSize + 20; dstFileName = (char*)malloc(dfnSize); if (dstFileName==NULL) EXM_THROW(71, "not enough memory for dstFileName"); }
if (sfnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0)
{
if (sfnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) {
DISPLAYLEVEL(1, "File extension doesn't match expected extension (%4s); will not process file: %s\n", suffix, srcFileName);
skippedFiles++;
continue;