diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index a66f0a49c..a5b66f7d3 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -497,14 +497,22 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v

Advanced streaming functions


 
 

Advanced Streaming compression functions

ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);   /**< size of CStream is variable, depending primarily on compression level */
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
-size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
-size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
 

+
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
+

start a new compression job, using same parameters from previous job. + This is typically useful to skip dictionary loading stage, since it will re-use it in-place.. + Note that zcs must be init at least once before using ZSTD_resetCStream(). + pledgedSrcSize==0 means "srcSize unknown". + If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + @return : 0, or an error code (which can be tested using ZSTD_isError()) +


+

Advanced Streaming decompression functions

typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
@@ -553,7 +561,7 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
-size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: fail if cdict==NULL. pledgedSrcSize can be 0, indicating unknown size.  For 0 size frames, use compressBegin_advanced */
 size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4ad978a4b..c0ccf83cd 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -31,7 +31,14 @@ typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZS /*-************************************* * Helper functions ***************************************/ +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG==1) +# include +#else +# define assert(condition) ((void)0) +#endif + #define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; } + size_t ZSTD_compressBound(size_t srcSize) { size_t const lowLimit = 256 KB; size_t const margin = (srcSize < lowLimit) ? (lowLimit-srcSize) >> 12 : 0; /* from 64 to 0 */ @@ -3031,10 +3038,13 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); } -size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize) +/* ZSTD_initCStream_internal() : + * params are supposed validated at this stage */ +static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) { + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); /* allocate buffers */ { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; if (zcs->inBuffSize < neededInBuffSize) { @@ -3068,11 +3078,19 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); } +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + CHECK_F( ZSTD_checkCParams(params.cParams) ); + return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize); +} + /* note : cdict must outlive compression session */ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) { ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict); - size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0); + size_t const initError = ZSTD_initCStream_internal(zcs, NULL, 0, params, 0); zcs->cdict = cdict; if (ZSTD_isError(initError)) return initError; return ZSTD_resetCStream_internal(zcs, 0); @@ -3081,14 +3099,14 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); - return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0); + return ZSTD_initCStream_internal(zcs, dict, dictSize, params, 0); } size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) { ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); if (pledgedSrcSize) params.fParams.contentSizeFlag = 1; - return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize); + return ZSTD_initCStream_internal(zcs, NULL, 0, params, pledgedSrcSize); } size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) diff --git a/tests/Makefile b/tests/Makefile index 809c90df5..70ef51878 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -26,7 +26,9 @@ PYTHON ?= python3 TESTARTEFACT := versionsTest namespaceTest -CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) +CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ + -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \ + -DZSTD_DEBUG=1 CFLAGS ?= -O3 CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 270c0221b..6f26ea555 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -207,6 +207,16 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s); } + /* Attempt bad compression parameters */ + DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++); + { size_t r; + ZSTD_parameters params = ZSTD_getParams(1, 0, 0); + params.cParams.searchLength = 2; + r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); + if (!ZSTD_isError(r)) goto _output_error; + DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r)); + } + /* skippable frame test */ DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);