mirror of
https://github.com/facebook/zstd.git
synced 2025-03-06 08:49:28 +02:00
Make pledgedSrcSize meaning clear for other functions
- Added tests - Moved new size functions to static link only
This commit is contained in:
Sean Purcell
parent
f07ddf88e8
commit
2db7249265
@ -344,8 +344,12 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
|
||||
{
|
||||
if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
|
||||
|
||||
|
||||
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
|
||||
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, pledgedSrcSize, ZSTDcrp_noMemset);
|
||||
{ ZSTD_parameters params = srcCCtx->params;
|
||||
params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
|
||||
ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
|
||||
}
|
||||
|
||||
/* copy tables */
|
||||
{ size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
|
||||
@ -2362,7 +2366,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
||||
U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
|
||||
U32 const checksumFlag = params.fParams.checksumFlag>0;
|
||||
U32 const windowSize = 1U << params.cParams.windowLog;
|
||||
U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize > pledgedSrcSize);
|
||||
U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
|
||||
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||
U32 const fcsCode = params.fParams.contentSizeFlag ?
|
||||
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */
|
||||
@ -2845,7 +2849,11 @@ static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
|
||||
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
|
||||
else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize));
|
||||
else {
|
||||
ZSTD_parameters params = cdict->refContext->params;
|
||||
params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
|
||||
CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -2939,7 +2947,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
|
||||
size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
|
||||
size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
|
||||
|
||||
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
||||
static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
|
||||
|
||||
@ -2957,6 +2965,14 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
||||
return 0; /* ready to go */
|
||||
}
|
||||
|
||||
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
|
||||
zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
|
||||
|
||||
return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
|
||||
}
|
||||
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize)
|
||||
@ -2988,7 +3004,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
|
||||
zcs->checksum = params.fParams.checksumFlag > 0;
|
||||
zcs->params = params;
|
||||
|
||||
return ZSTD_resetCStream(zcs, pledgedSrcSize);
|
||||
return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
|
||||
}
|
||||
|
||||
/* note : cdict must outlive compression session */
|
||||
|
||||
88
lib/zstd.h
88
lib/zstd.h
@ -88,44 +88,12 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
|
||||
ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
|
||||
const void* src, size_t compressedSize);
|
||||
|
||||
#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
|
||||
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
||||
|
||||
/*! ZSTD_getFrameContentSize() :
|
||||
* `src` should point to the start of a ZSTD encoded frame
|
||||
* @return : decompressed size of the frame pointed to be `src` if known, otherwise
|
||||
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
|
||||
* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */
|
||||
ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
|
||||
|
||||
/*! ZSTD_findDecompressedSize() :
|
||||
* `src` should point the start of a series of ZSTD encoded and/or skippable frames
|
||||
* `srcSize` must be the _exact_ size of this series
|
||||
* (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
|
||||
* @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
|
||||
* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
|
||||
* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
|
||||
*
|
||||
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
|
||||
* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
|
||||
* In which case, it's necessary to use streaming mode to decompress data.
|
||||
* Optionally, application can still use ZSTD_decompress() while relying on implied limits.
|
||||
* (For example, data may be necessarily cut into blocks <= 16 KB).
|
||||
* note 2 : decompressed size is always present when compression is done with ZSTD_compress()
|
||||
* note 3 : decompressed size can be very large (64-bits value),
|
||||
* potentially larger than what local system can handle as a single memory segment.
|
||||
* In which case, it's necessary to use streaming mode to decompress data.
|
||||
* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
|
||||
* Always ensure result fits within application's authorized limits.
|
||||
* Each application can set its own limits.
|
||||
* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
|
||||
* read each contained frame header. This is efficient as most of the data is skipped,
|
||||
* however it does mean that all frame data must be present and valid. */
|
||||
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
|
||||
|
||||
/*! ZSTD_getDecompressedSize() :
|
||||
* WARNING: This function is now obsolete. ZSTD_findDecompressedSize should be used,
|
||||
* or if only exactly one ZSTD frame is needed, ZSTD_getFrameContentSize can be used.
|
||||
* NOTE: This function is planned to be obsolete, in favour of ZSTD_findDecompressedSize,
|
||||
* or for single frames, ZSTD_getFrameContentSize. Both of the new functions return more
|
||||
* descriptive return values, disambiguating empty frames from size unknown or errors.
|
||||
* Additionally, ZSTD_findDecompressedSize handles multi-frame inputs, returning the total
|
||||
* size of all frames put together.
|
||||
*
|
||||
* 'src' is the start of a zstd compressed frame.
|
||||
* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
|
||||
@ -369,6 +337,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
|
||||
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
|
||||
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
|
||||
|
||||
#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
|
||||
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
||||
|
||||
#define ZSTD_WINDOWLOG_MAX_32 25
|
||||
#define ZSTD_WINDOWLOG_MAX_64 27
|
||||
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
|
||||
@ -422,6 +393,41 @@ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
||||
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
||||
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
||||
|
||||
/***************************************
|
||||
* Decompressed size functions
|
||||
***************************************/
|
||||
/*! ZSTD_getFrameContentSize() :
|
||||
* `src` should point to the start of a ZSTD encoded frame
|
||||
* @return : decompressed size of the frame pointed to be `src` if known, otherwise
|
||||
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
|
||||
* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */
|
||||
ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
|
||||
|
||||
/*! ZSTD_findDecompressedSize() :
|
||||
* `src` should point the start of a series of ZSTD encoded and/or skippable frames
|
||||
* `srcSize` must be the _exact_ size of this series
|
||||
* (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
|
||||
* @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
|
||||
* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
|
||||
* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
|
||||
*
|
||||
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
|
||||
* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
|
||||
* In which case, it's necessary to use streaming mode to decompress data.
|
||||
* Optionally, application can still use ZSTD_decompress() while relying on implied limits.
|
||||
* (For example, data may be necessarily cut into blocks <= 16 KB).
|
||||
* note 2 : decompressed size is always present when compression is done with ZSTD_compress()
|
||||
* note 3 : decompressed size can be very large (64-bits value),
|
||||
* potentially larger than what local system can handle as a single memory segment.
|
||||
* In which case, it's necessary to use streaming mode to decompress data.
|
||||
* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
|
||||
* Always ensure result fits within application's authorized limits.
|
||||
* Each application can set its own limits.
|
||||
* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
|
||||
* read each contained frame header. This is efficient as most of the data is skipped,
|
||||
* however it does mean that all frame data must be present and valid. */
|
||||
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/***************************************
|
||||
* Advanced compression functions
|
||||
@ -555,12 +561,12 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
||||
|
||||
/*===== Advanced Streaming compression functions =====*/
|
||||
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
|
||||
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
|
||||
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
|
||||
|
||||
|
||||
@ -617,8 +623,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
|
||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize);
|
||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
@ -27,7 +27,7 @@ TESTARTEFACT := versionsTest namespaceTest
|
||||
|
||||
|
||||
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
|
||||
CFLAGS ?= -g
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
|
||||
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
|
||||
@ -438,6 +438,34 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
|
||||
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
|
||||
|
||||
/* Unknown srcSize */
|
||||
DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++);
|
||||
{ ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); } /* cstream advanced should write the 0 size field */
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = 0;
|
||||
inBuff.pos = 0;
|
||||
outBuff.dst = compressedBuffer;
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
|
||||
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
|
||||
cSize = outBuff.pos;
|
||||
if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
|
||||
|
||||
ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = 0;
|
||||
inBuff.pos = 0;
|
||||
outBuff.dst = compressedBuffer;
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
|
||||
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
|
||||
cSize = outBuff.pos;
|
||||
if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
_end:
|
||||
FUZ_freeDictionary(dictionary);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user