mirror of
https://github.com/facebook/zstd.git
synced 2025-03-06 16:56:49 +02:00
Merge pull request #678 from facebook/apiChange
Breaking API Change around CDict
This commit is contained in:
Yann Collet
GitHub
commit
a51cab6e68
@ -400,7 +400,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
|
||||
ZSTD_parameters params, ZSTD_customMem customMem);
|
||||
ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
|
||||
</b><p> Create a ZSTD_CDict using external alloc and free, and customized compression parameters
|
||||
</p></pre><BR>
|
||||
|
||||
@ -427,12 +427,19 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||
both values are optional, select `0` if unknown.
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize,
|
||||
ZSTD_parameters params);
|
||||
</b><p> Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter
|
||||
<pre><b>size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize,
|
||||
ZSTD_parameters params);
|
||||
</b><p> Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_CDict* cdict, ZSTD_frameParameters fParams);
|
||||
</b><p> Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter15"></a><h2>Advanced decompression functions</h2><pre></pre>
|
||||
@ -504,6 +511,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
|
||||
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
|
||||
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams); </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */<b>
|
||||
</pre></b><BR>
|
||||
<pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
|
||||
</b><p> start a new compression job, using same parameters from previous job.
|
||||
@ -561,7 +569,7 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
||||
<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
|
||||
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); </b>/**< note: fail if cdict==NULL. pledgedSrcSize can be 0, indicating unknown size. For 0 size frames, use compressBegin_advanced */<b>
|
||||
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
|
||||
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */<b>
|
||||
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */<b>
|
||||
</pre></b><BR>
|
||||
|
||||
@ -16,9 +16,9 @@
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define FORCE_INLINE static __forceinline
|
||||
# include <intrin.h> /* For Visual 2005 */
|
||||
# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
|
||||
# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
|
||||
#else
|
||||
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# ifdef __GNUC__
|
||||
|
||||
@ -20,6 +20,26 @@
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Debug
|
||||
***************************************/
|
||||
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
|
||||
# include <assert.h>
|
||||
#else
|
||||
# define assert(condition) ((void)0)
|
||||
#endif
|
||||
|
||||
#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
|
||||
|
||||
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
|
||||
# include <stdio.h>
|
||||
static unsigned g_debugLevel = ZSTD_DEBUG;
|
||||
# define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); }
|
||||
#else
|
||||
# define DEBUGLOG(l, ...) {} /* disabled */
|
||||
#endif
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
@ -38,14 +58,6 @@ static size_t const entropyScratchSpace_size = HUF_WORKSPACE_SIZE;
|
||||
/*-*************************************
|
||||
* Helper functions
|
||||
***************************************/
|
||||
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG==1)
|
||||
# include <assert.h>
|
||||
#else
|
||||
# define assert(condition) ((void)0)
|
||||
#endif
|
||||
|
||||
#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
|
||||
|
||||
size_t ZSTD_compressBound(size_t srcSize) {
|
||||
size_t const lowLimit = 256 KB;
|
||||
size_t const margin = (srcSize < lowLimit) ? (lowLimit-srcSize) >> 12 : 0; /* from 64 to 0 */
|
||||
@ -407,6 +419,7 @@ size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx,
|
||||
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
|
||||
{ ZSTD_parameters params = srcCCtx->params;
|
||||
params.fParams = fParams;
|
||||
DEBUGLOG(5, "ZSTD_resetCCtx_internal : dictIDFlag : %u \n", !fParams.noDictIDFlag);
|
||||
ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
|
||||
}
|
||||
|
||||
@ -887,7 +900,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
|
||||
const U32 pos = (U32)((const BYTE*)literals - g_start);
|
||||
if (g_start==NULL) g_start = (const BYTE*)literals;
|
||||
if ((pos > 1895000) && (pos < 1895300))
|
||||
fprintf(stderr, "Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
|
||||
DEBUGLOG(5, "Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
|
||||
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
|
||||
}
|
||||
#endif
|
||||
@ -1049,7 +1062,6 @@ static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
|
||||
{
|
||||
switch(mls)
|
||||
{
|
||||
//case 3: return ZSTD_hash3Ptr(p, hBits);
|
||||
default:
|
||||
case 4: return ZSTD_hash4Ptr(p, hBits);
|
||||
case 5: return ZSTD_hash5Ptr(p, hBits);
|
||||
@ -2503,7 +2515,8 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
|
||||
{ BYTE* const op = (BYTE*)dst;
|
||||
U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
|
||||
U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
|
||||
U32 const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */
|
||||
U32 const checksumFlag = params.fParams.checksumFlag>0;
|
||||
U32 const windowSize = 1U << params.cParams.windowLog;
|
||||
U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
|
||||
@ -2515,6 +2528,9 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
||||
size_t pos;
|
||||
|
||||
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
|
||||
DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u \n", !params.fParams.noDictIDFlag);
|
||||
DEBUGLOG(5, "ZSTD_writeFrameHeader : dictID : %u \n", dictID);
|
||||
DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDSizeCode : %u \n", dictIDSizeCode);
|
||||
|
||||
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
|
||||
op[4] = frameHeaderDecriptionByte; pos=5;
|
||||
@ -2932,8 +2948,16 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
|
||||
return ZSTD_sizeof_CCtx(cdict->refContext) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict);
|
||||
}
|
||||
|
||||
static ZSTD_parameters ZSTD_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
params.cParams = cParams;
|
||||
params.fParams = fParams;
|
||||
return params;
|
||||
}
|
||||
|
||||
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference,
|
||||
ZSTD_parameters params, ZSTD_customMem customMem)
|
||||
ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
|
||||
{
|
||||
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
|
||||
if (!customMem.customAlloc || !customMem.customFree) return NULL;
|
||||
@ -2958,7 +2982,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
|
||||
cdict->dictContent = internalBuffer;
|
||||
}
|
||||
|
||||
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
|
||||
{ ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */
|
||||
ZSTD_parameters const params = ZSTD_makeParams(cParams, fParams);
|
||||
size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
|
||||
if (ZSTD_isError(errorCode)) {
|
||||
ZSTD_free(cdict->dictBuffer, customMem);
|
||||
ZSTD_free(cdict, customMem);
|
||||
@ -2975,17 +3001,15 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
|
||||
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_customMem const allocator = { NULL, NULL, NULL };
|
||||
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
return ZSTD_createCDict_advanced(dict, dictSize, 0, params, allocator);
|
||||
ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
return ZSTD_createCDict_advanced(dict, dictSize, 0, cParams, allocator);
|
||||
}
|
||||
|
||||
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_customMem const allocator = { NULL, NULL, NULL };
|
||||
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
return ZSTD_createCDict_advanced(dict, dictSize, 1, params, allocator);
|
||||
ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
return ZSTD_createCDict_advanced(dict, dictSize, 1, cParams, allocator);
|
||||
}
|
||||
|
||||
size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
|
||||
@ -3010,6 +3034,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
|
||||
ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
|
||||
{
|
||||
if (cdict==NULL) return ERROR(GENERIC); /* does not support NULL cdict */
|
||||
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced : dictIDFlag == %u \n", !fParams.noDictIDFlag);
|
||||
if (cdict->dictContentSize)
|
||||
CHECK_F( ZSTD_copyCCtx_internal(cctx, cdict->refContext, fParams, pledgedSrcSize) )
|
||||
else {
|
||||
@ -3023,11 +3048,20 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
|
||||
/* ZSTD_compressBegin_usingCDict() :
|
||||
* pledgedSrcSize=0 means "unknown"
|
||||
* if pledgedSrcSize>0, it will enable contentSizeFlag */
|
||||
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize)
|
||||
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
|
||||
{
|
||||
ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
|
||||
fParams.contentSizeFlag = (pledgedSrcSize > 0);
|
||||
return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, pledgedSrcSize);
|
||||
ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
|
||||
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u \n", !fParams.noDictIDFlag);
|
||||
return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0);
|
||||
}
|
||||
|
||||
size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
|
||||
{
|
||||
CHECK_F (ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize)); /* will check if cdict != NULL */
|
||||
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
/*! ZSTD_compress_usingCDict() :
|
||||
@ -3041,8 +3075,7 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
|
||||
const ZSTD_CDict* cdict)
|
||||
{
|
||||
ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
|
||||
CHECK_F (ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize)); /* will check if cdict != NULL */
|
||||
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
|
||||
return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams);
|
||||
}
|
||||
|
||||
|
||||
@ -3127,6 +3160,8 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long p
|
||||
{
|
||||
if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
|
||||
|
||||
DEBUGLOG(5, "ZSTD_resetCStream_internal : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag);
|
||||
|
||||
if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs->cctx, zcs->cdict, zcs->params.fParams, pledgedSrcSize))
|
||||
else CHECK_F(ZSTD_compressBegin_internal(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
|
||||
|
||||
@ -3144,6 +3179,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
|
||||
zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
|
||||
DEBUGLOG(5, "ZSTD_resetCStream : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag);
|
||||
return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
|
||||
}
|
||||
|
||||
@ -3179,18 +3215,27 @@ static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
|
||||
zcs->checksum = params.fParams.checksumFlag > 0;
|
||||
zcs->params = params;
|
||||
|
||||
DEBUGLOG(5, "ZSTD_initCStream_stage2 : dictIDFlag == %u \n", !params.fParams.noDictIDFlag);
|
||||
return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
|
||||
}
|
||||
|
||||
/* ZSTD_initCStream_usingCDict_advanced() :
|
||||
* same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
|
||||
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams)
|
||||
{
|
||||
if (!cdict) return ERROR(GENERIC); /* cannot handle NULL cdict (does not know what to do) */
|
||||
{ ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
|
||||
params.fParams = fParams;
|
||||
zcs->cdict = cdict;
|
||||
return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize);
|
||||
}
|
||||
}
|
||||
|
||||
/* note : cdict must outlive compression session */
|
||||
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
|
||||
{
|
||||
if (!cdict) return ERROR(GENERIC); /* cannot handle NULL cdict (does not know what to do) */
|
||||
{ ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
zcs->cdict = cdict;
|
||||
return ZSTD_initCStream_stage2(zcs, params, 0);
|
||||
}
|
||||
ZSTD_frameParameters const fParams = { 0 /* content */, 0 /* checksum */, 0 /* noDictID */ };
|
||||
return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, 0, fParams);
|
||||
}
|
||||
|
||||
static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
|
||||
@ -3202,11 +3247,12 @@ static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
|
||||
|
||||
if (dict && dictSize >= 8) {
|
||||
ZSTD_freeCDict(zcs->cdictLocal);
|
||||
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* copy */, params, zcs->customMem);
|
||||
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* copy */, params.cParams, zcs->customMem);
|
||||
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
|
||||
zcs->cdict = zcs->cdictLocal;
|
||||
}
|
||||
|
||||
DEBUGLOG(5, "ZSTD_initCStream_internal : dictIDFlag == %u \n", !params.fParams.noDictIDFlag);
|
||||
return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize);
|
||||
}
|
||||
|
||||
@ -3215,6 +3261,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
CHECK_F( ZSTD_checkCParams(params.cParams) );
|
||||
DEBUGLOG(5, "ZSTD_initCStream_advanced : dictIDFlag == %u \n", !params.fParams.noDictIDFlag);
|
||||
return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize);
|
||||
}
|
||||
|
||||
|
||||
@ -521,7 +521,7 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
|
||||
if (updateDict) {
|
||||
ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL;
|
||||
if (dict && dictSize) {
|
||||
zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params, cmem);
|
||||
zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params.cParams, cmem);
|
||||
if (zcs->cdict == NULL) return ERROR(memory_allocation);
|
||||
} }
|
||||
zcs->frameContentSize = pledgedSrcSize;
|
||||
|
||||
26
lib/zstd.h
26
lib/zstd.h
@ -488,7 +488,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, siz
|
||||
/*! ZSTD_createCDict_advanced() :
|
||||
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
|
||||
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
|
||||
ZSTD_parameters params, ZSTD_customMem customMem);
|
||||
ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
|
||||
|
||||
/*! ZSTD_sizeof_CDict() :
|
||||
* Gives the amount of memory used by a given ZSTD_sizeof_CDict */
|
||||
@ -514,12 +514,19 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
||||
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_compress_advanced() :
|
||||
* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
|
||||
ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize,
|
||||
ZSTD_parameters params);
|
||||
* Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter */
|
||||
ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize,
|
||||
ZSTD_parameters params);
|
||||
|
||||
/*! ZSTD_compress_usingCDict_advanced() :
|
||||
* Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
|
||||
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_CDict* cdict, ZSTD_frameParameters fParams);
|
||||
|
||||
|
||||
/*--- Advanced decompression functions ---*/
|
||||
@ -579,7 +586,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
|
||||
* Note : this use case also happens when using a non-conformant dictionary.
|
||||
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
|
||||
* - This is not a Zstandard frame.
|
||||
* When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
|
||||
* When identifying the exact failure cause, it's possible to use ZSTD_getFrameParams(), which will provide a more precise error code. */
|
||||
ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
||||
|
||||
|
||||
@ -595,6 +602,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dic
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
|
||||
|
||||
/*! ZSTD_resetCStream() :
|
||||
* start a new compression job, using same parameters from previous job.
|
||||
@ -659,7 +667,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: fail if cdict==NULL. pledgedSrcSize can be 0, indicating unknown size. For 0 size frames, use compressBegin_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */
|
||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
|
||||
|
||||
|
||||
@ -274,7 +274,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
|
||||
if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
|
||||
if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
|
||||
cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem);
|
||||
cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams.cParams, cmem);
|
||||
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
|
||||
do {
|
||||
U32 blockNb;
|
||||
|
||||
@ -105,6 +105,7 @@ static clock_t g_time = 0;
|
||||
#undef MIN
|
||||
#define MIN(a,b) ((a) < (b) ? (a) : (b))
|
||||
|
||||
|
||||
/* ************************************************************
|
||||
* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
|
||||
***************************************************************/
|
||||
|
||||
@ -405,14 +405,13 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
|
||||
{ ZSTD_parameters params = ZSTD_getParams(1, CNBuffSize, dictSize);
|
||||
{ ZSTD_customMem customMem = { NULL, NULL, NULL };
|
||||
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, params, customMem);
|
||||
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
|
||||
ZSTD_customMem customMem = { NULL, NULL, NULL };
|
||||
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
|
||||
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
CNBuffer, CNBuffSize, cdict);
|
||||
ZSTD_freeCDict(cdict);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
}
|
||||
ZSTD_freeCDict(cdict);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
@ -430,7 +429,33 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
|
||||
DISPLAYLEVEL(4, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
|
||||
{ ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
|
||||
ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
|
||||
ZSTD_customMem const customMem = { NULL, NULL, NULL };
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
|
||||
cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
CNBuffer, CNBuffSize, cdict, fParams);
|
||||
ZSTD_freeCDict(cdict);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : try retrieving contentSize from frame : ", testNb++);
|
||||
{ U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
|
||||
if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK (unknown)\n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : frame built without dictID should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
|
||||
{ ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
|
||||
p.fParams.noDictIDFlag = 1;
|
||||
cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
@ -697,6 +722,14 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
||||
}
|
||||
|
||||
|
||||
static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
params.cParams = cParams;
|
||||
params.fParams = fParams;
|
||||
return params;
|
||||
}
|
||||
|
||||
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
|
||||
{
|
||||
size_t const lengthMask = ((size_t)1 << logLength) - 1;
|
||||
@ -760,7 +793,6 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
|
||||
size_t sampleSize, maxTestSize, totalTestSize;
|
||||
size_t cSize, totalCSize, totalGenSize;
|
||||
XXH64_state_t xxhState;
|
||||
U64 crcOrig;
|
||||
BYTE* sampleBuffer;
|
||||
const BYTE* dict;
|
||||
@ -917,22 +949,22 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
|
||||
} else {
|
||||
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
|
||||
ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
|
||||
ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
|
||||
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
|
||||
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
|
||||
ZSTD_parameters p;
|
||||
size_t errorCode;
|
||||
p.cParams = cPar; p.fParams = fpar;
|
||||
errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
|
||||
ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
|
||||
size_t const errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
|
||||
}
|
||||
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx, 0);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode));
|
||||
} }
|
||||
XXH64_reset(&xxhState, 0);
|
||||
ZSTD_setCCtxParameter(ctx, ZSTD_p_forceWindow, FUZ_rand(&lseed) & 1);
|
||||
|
||||
{ U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
|
||||
U32 n;
|
||||
XXH64_state_t xxhState;
|
||||
XXH64_reset(&xxhState, 0);
|
||||
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
|
||||
size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
|
||||
@ -947,13 +979,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
|
||||
memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
|
||||
totalTestSize += segmentSize;
|
||||
} }
|
||||
}
|
||||
|
||||
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
|
||||
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
|
||||
cSize += flushResult;
|
||||
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
|
||||
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
|
||||
cSize += flushResult;
|
||||
}
|
||||
crcOrig = XXH64_digest(&xxhState);
|
||||
}
|
||||
crcOrig = XXH64_digest(&xxhState);
|
||||
|
||||
/* streaming decompression test */
|
||||
if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
|
||||
|
||||
@ -448,14 +448,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
|
||||
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : dictionary compression with masked dictID : ", testNb++);
|
||||
{ ZSTD_parameters params = ZSTD_getParams(1, CNBufferSize, dictionary.filled);
|
||||
ZSTD_CDict* cdict;
|
||||
params.fParams.noDictIDFlag = 1;
|
||||
params.fParams.contentSizeFlag = 1; /* test contentSize, should be disabled with initCStream_usingCDict */
|
||||
cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, 1, params, customMem);
|
||||
{ size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
|
||||
if (ZSTD_isError(initError)) goto _output_error; }
|
||||
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
|
||||
ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, 1 /* byReference */, cParams, customMem);
|
||||
size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, CNBufferSize, fParams);
|
||||
if (ZSTD_isError(initError)) goto _output_error;
|
||||
cSize = 0;
|
||||
outBuff.dst = compressedBuffer;
|
||||
outBuff.size = compressedBufferSize;
|
||||
@ -473,6 +471,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++);
|
||||
{ U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
|
||||
if (did != 0) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK (not detected) \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
|
||||
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
|
||||
if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
|
||||
|
||||
@ -236,7 +236,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
|
||||
if (compressor == BMK_ZSTD) {
|
||||
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
|
||||
ZSTD_customMem const cmem = { NULL, NULL, NULL };
|
||||
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem);
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams.cParams, cmem);
|
||||
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
|
||||
|
||||
do {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user