krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-07 01:10:04 +02:00
Code Issues Releases 1 Activity

Added : zstd buffered API

Browse Source
This commit is contained in:
Yann Collet 2015-11-25 14:42:45 +01:00
parent c36521571e
commit 88fcd2916e
14 changed files with 1177 additions and 500 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Makefile
View File

@ -37,7 +37,14 @@ export VERSION := 0.4.0
PRGDIR = programs PRGDIR = programs
ZSTDDIR = lib ZSTDDIR = lib
.PHONY: clean # Define nul output
ifneq (,$(filter Windows%,$(OS)))
VOID = nul
else
VOID = /dev/null
endif
.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
default: zstdprogram default: zstdprogram
@ -49,8 +56,8 @@ zstdprogram:
$(MAKE) -C $(PRGDIR) $(MAKE) -C $(PRGDIR)
clean: clean:
$(MAKE) -C $(ZSTDDIR) $@ @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
$(MAKE) -C $(PRGDIR) $@ @$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@echo Cleaning completed @echo Cleaning completed

2
NEWS
View File

@ -1,5 +1,7 @@
v0.4.0 v0.4.0
Command line utility is now compatible with high compression levels
Removed zstdhc => merged into zstd Removed zstdhc => merged into zstd
Added : ZBUFF API (see zstd_buffered.h)
Rolling buffer support Rolling buffer support
v0.3.6 v0.3.6

2
lib/error.h
View File

@ -68,7 +68,7 @@ extern "C" {
#define ERROR_LIST(ITEM) \ #define ERROR_LIST(ITEM) \
ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \ ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
ITEM(PREFIX(mode_unsupported)) \ ITEM(PREFIX(mode_unsupported)) ITEM(PREFIX(init_missing))\
ITEM(PREFIX(memory_allocation)) \ ITEM(PREFIX(memory_allocation)) \
ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \ ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \ ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \

531
lib/zstd_buffered.c Normal file
View File

@ -0,0 +1,531 @@
/*
Buffered version of Zstd compression library
Copyright (C) 2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/* The objects defined into this file should be considered experimental.
* They are not labelled stable, as their prototype may change in the future.
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
*/
/* *************************************
* Includes
***************************************/
#include <stdlib.h>
#include "error.h"
#include "zstd_static.h"
#include "zstd_buffered_static.h"
/** ************************************************
* Streaming compression
*
* A ZBUFF_CCtx object is required to track streaming operation.
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
* Use ZBUFF_compressInit() to start a new compression operation.
* ZBUFF_CCtx objects can be reused multiple times.
*
* Use ZBUFF_compressContinue() repetitively to consume your input.
* *srcSizePtr and *maxDstSizePtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
* or an error code, which can be tested using ZBUFF_isError().
*
* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
* Note that it will not output more than *maxDstSizePtr.
* Therefore, some content might still be left into its internal buffer if dst buffer is too small.
* @return : nb of bytes still present into internal buffer (0 if it's empty)
* or an error code, which can be tested using ZBUFF_isError().
*
* ZBUFF_compressEnd() instructs to finish a frame.
* It will perform a flush and write frame epilogue.
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small.
* @return : nb of bytes still present into internal buffer (0 if it's empty)
* or an error code, which can be tested using ZBUFF_isError().
*
* Hint : recommended buffer sizes (not compulsory)
* input : 128 KB block size is the internal unit, it improves latency to use this value.
* output : ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block at best speed.
* **************************************************/
typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
/* *** Ressources *** */
struct ZBUFF_CCtx_s {
ZSTD_CCtx* zc;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
size_t inBuffPos;
size_t inBuffTarget;
size_t blockSize;
char* outBuff;
size_t outBuffSize;
size_t outBuffContentSize;
size_t outBuffFlushedSize;
ZBUFF_cStage stage;
}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
ZBUFF_CCtx* ZBUFF_createCCtx(void)
{
ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx));
memset(zbc, 0, sizeof(*zbc));
zbc->zc = ZSTD_createCCtx();
return zbc;
}
size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
{
if (zbc==NULL) return 0; /* support free on NULL */
ZSTD_freeCCtx(zbc->zc);
free(zbc);
return 0;
}
/* *** Initialization *** */
#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) )
#define BLOCKSIZE (128 * 1024) /* a bit too "magic", should come from reference */
size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, ZSTD_parameters params)
{
size_t neededInBuffSize;
ZSTD_validateParams(&params);
neededInBuffSize = (size_t)1 << params.windowLog;
/* allocate buffers */
if (zbc->inBuffSize < neededInBuffSize)
{
zbc->inBuffSize = neededInBuffSize;
free(zbc->inBuff); /* should not be necessary */
zbc->inBuff = (char*)malloc(neededInBuffSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
}
zbc->blockSize = MIN(BLOCKSIZE, zbc->inBuffSize);
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1)
{
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
free(zbc->outBuff); /* should not be necessary */
zbc->outBuff = (char*)malloc(zbc->outBuffSize);
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
}
zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, zbc->outBuff, zbc->outBuffSize, params);
if (ZSTD_isError(zbc->outBuffContentSize)) return zbc->outBuffContentSize;
zbc->inToCompress = 0;
zbc->inBuffPos = 0;
zbc->inBuffTarget = zbc->blockSize;
zbc->outBuffFlushedSize = 0;
zbc->stage = ZBUFFcs_flush; /* starts by flushing the header */
return 0; /* ready to go */
}
size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
{
return ZBUFF_compressInit_advanced(zbc, ZSTD_getParams(compressionLevel, 0));
}
/* *** Compression *** */
static size_t ZBUFF_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
size_t length = MIN(maxDstSize, srcSize);
memcpy(dst, src, length);
return length;
}
static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
void* dst, size_t* maxDstSizePtr,
const void* src, size_t* srcSizePtr,
int flush) /* aggregate : wait for full block before compressing */
{
U32 notDone = 1;
const char* const istart = (const char*)src;
const char* ip = istart;
const char* const iend = istart + *srcSizePtr;
char* const ostart = (char*)dst;
char* op = ostart;
char* const oend = ostart + *maxDstSizePtr;
while (notDone)
{
switch(zbc->stage)
{
case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
case ZBUFFcs_load:
/* complete inBuffer */
{
size_t toLoad = zbc->inBuffTarget - zbc->inBuffPos;
size_t loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
zbc->inBuffPos += loaded;
ip += loaded;
if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) )
{ notDone = 0; break; } /* not enough input to get a full block : stop there, wait for more */
}
/* compress current block (note : this stage cannot be stopped in the middle) */
{
void* cDst;
size_t cSize;
size_t iSize = zbc->inBuffPos - zbc->inToCompress;
if ((size_t)(oend-op) > ZSTD_compressBound(iSize))
cDst = op; /* compress directly into output buffer (avoid flush stage) */
else
cDst = zbc->outBuff;
cSize = ZSTD_compressContinue(zbc->zc, cDst, oend-op, zbc->inBuff + zbc->inToCompress, iSize);
if (ZSTD_isError(cSize)) return cSize;
/* prepare next block */
zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
if (zbc->inBuffTarget > zbc->inBuffSize)
{ zbc->inBuffPos = 0; zbc->inBuffTarget = zbc->blockSize; }
zbc->inToCompress = zbc->inBuffPos;
if (cDst == op) { op += cSize; break; } /* no need to flush */
zbc->outBuffContentSize = cSize;
zbc->outBuffFlushedSize = 0;
zbc->stage = ZBUFFcs_flush;
// break; /* flush stage follows */
}
case ZBUFFcs_flush:
/* flush into dst */
{
size_t toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
size_t flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
op += flushed;
zbc->outBuffFlushedSize += flushed;
if (toFlush!=flushed)
{ notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
zbc->outBuffContentSize = 0;
zbc->outBuffFlushedSize = 0;
zbc->stage = ZBUFFcs_load;
break;
}
}
}
*srcSizePtr = ip - istart;
*maxDstSizePtr = op - ostart;
{
size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
if (hintInSize==0) hintInSize = zbc->blockSize;
return hintInSize;
}
}
size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
void* dst, size_t* maxDstSizePtr,
const void* src, size_t* srcSizePtr)
{ return ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, src, srcSizePtr, 0); }
/* *** Finalize *** */
size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
{
size_t srcSize = 0;
ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, NULL, &srcSize, 1);
return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
}
size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
{
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
BYTE* const oend = ostart + *maxDstSizePtr;
size_t outSize = *maxDstSizePtr;
size_t epilogueSize, remaining;
ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
op += outSize;
epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */
zbc->outBuffContentSize += epilogueSize;
outSize = oend-op;
zbc->stage = ZBUFFcs_flush;
remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */
op += outSize;
if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */
*maxDstSizePtr = op-ostart; /* tells how many bytes were written */
return remaining;
}
/** ************************************************
* Streaming decompression
*
* A ZBUFF_DCtx object is required to track streaming operation.
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
* Use ZBUFF_decompressInit() to start a new decompression operation.
* ZBUFF_DCtx objects can be reused multiple times.
*
* Use ZBUFF_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *maxDstSizePtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
* or 0 when a frame is completely decoded
* or an error code, which can be tested using ZBUFF_isError().
*
* Hint : recommended buffer sizes (not compulsory)
* output : 128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
* input : just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* **************************************************/
typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader,
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
/* *** Resource management *** */
#define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */
struct ZBUFF_DCtx_s {
ZSTD_DCtx* zc;
ZSTD_parameters params;
char* inBuff;
size_t inBuffSize;
size_t inPos;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t hPos;
ZBUFF_dStage stage;
unsigned char headerBuffer[ZSTD_frameHeaderSize_max];
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
ZBUFF_DCtx* ZBUFF_createDCtx(void)
{
ZBUFF_DCtx* zbc = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx));
memset(zbc, 0, sizeof(*zbc));
zbc->zc = ZSTD_createDCtx();
zbc->stage = ZBUFFds_init;
return zbc;
}
size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc)
{
if (zbc==NULL) return 0; /* support free on null */
ZSTD_freeDCtx(zbc->zc);
free(zbc);
return 0;
}
/* *** Initialization *** */
size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc)
{
zbc->stage = ZBUFFds_readHeader;
zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0;
return ZSTD_resetDCtx(zbc->zc);
}
/* *** Decompression *** */
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr)
{
const char* const istart = (const char*)src;
const char* ip = istart;
const char* const iend = istart + *srcSizePtr;
char* const ostart = (char*)dst;
char* op = ostart;
char* const oend = ostart + *maxDstSizePtr;
U32 notDone = 1;
while (notDone)
{
switch(zbc->stage)
{
case ZBUFFds_init :
return ERROR(init_missing);
case ZBUFFds_readHeader :
/* read header from src */
{
size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr);
if (ZSTD_isError(headerSize)) return headerSize;
if (headerSize)
{
/* not enough input to decode header : tell how many bytes would be necessary */
memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr);
zbc->hPos += *srcSizePtr;
*maxDstSizePtr = 0;
zbc->stage = ZBUFFds_loadHeader;
return headerSize - zbc->hPos;
}
zbc->stage = ZBUFFds_decodeHeader;
break;
}
case ZBUFFds_loadHeader:
/* complete header from src */
{
size_t headerSize = ZBUFF_limitCopy(
zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos,
src, *srcSizePtr);
zbc->hPos += headerSize;
ip += headerSize;
headerSize = ZSTD_getFrameParams(&(zbc->params), zbc->headerBuffer, zbc->hPos);
if (ZSTD_isError(headerSize)) return headerSize;
if (headerSize)
{
/* not enough input to decode header : tell how many bytes would be necessary */
*maxDstSizePtr = 0;
return headerSize - zbc->hPos;
}
// zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */
}
case ZBUFFds_decodeHeader:
/* apply header to create / resize buffers */
{
size_t neededOutSize = (size_t)1 << zbc->params.windowLog;
size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
if (zbc->inBuffSize < neededInSize)
{
free(zbc->inBuff);
zbc->inBuffSize = neededInSize;
zbc->inBuff = (char*)malloc(neededInSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
}
if (zbc->outBuffSize < neededOutSize)
{
free(zbc->outBuff);
zbc->outBuffSize = neededOutSize;
zbc->outBuff = (char*)malloc(neededOutSize);
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
}
}
memcpy(zbc->inBuff, zbc->headerBuffer, zbc->hPos);
zbc->inPos = zbc->hPos;
zbc->hPos = 0;
zbc->stage = ZBUFFds_load;
break; /* useless : stage follows */
case ZBUFFds_read:
{
size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
if (neededInSize==0) /* end of frame */
{
zbc->stage = ZBUFFds_init;
notDone = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize)
{
/* directly decode from src */
size_t decodedSize = ZSTD_decompressContinue(zbc->zc,
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
ip += neededInSize;
if (!decodedSize) break; /* this was just a header */
zbc->outEnd = zbc->outStart + decodedSize;
zbc->stage = ZBUFFds_flush;
break;
}
if (ip==iend) { notDone = 0; break; } /* no more input */
zbc->stage = ZBUFFds_load;
}
case ZBUFFds_load:
{
size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
size_t toLoad = neededInSize - zbc->inPos;
size_t loadedSize;
if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, zbc->inBuffSize - zbc->inPos, ip, iend-ip);
ip += loadedSize;
zbc->inPos += loadedSize;
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
{
size_t decodedSize = ZSTD_decompressContinue(zbc->zc,
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
zbc->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
zbc->inPos = 0; /* input is consumed */
if (!decodedSize) { zbc->stage = ZBUFFds_read; break; } /* this was just a header */
zbc->outEnd = zbc->outStart + decodedSize;
zbc->stage = ZBUFFds_flush;
// break; /* ZBUFFds_flush follows */
}
}
case ZBUFFds_flush:
{
size_t toFlushSize = zbc->outEnd - zbc->outStart;
size_t flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);
op += flushedSize;
zbc->outStart += flushedSize;
if (flushedSize == toFlushSize)
{
zbc->stage = ZBUFFds_read;
if (zbc->outStart + BLOCKSIZE > zbc->outBuffSize)
zbc->outStart = zbc->outEnd = 0;
}
break;
}
}
}
*srcSizePtr = ip-istart;
*maxDstSizePtr = op-ostart;
return ZSTD_nextSrcSizeToDecompress(zbc->zc) - zbc->inPos;
}
/* *************************************
* Tool functions
***************************************/
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
size_t ZBUFF_recommendedCInSize() { return BLOCKSIZE; }
size_t ZBUFF_recommendedCOutSize() { return ZSTD_compressBound(BLOCKSIZE) + 6; }
size_t ZBUFF_recommendedDInSize() { return BLOCKSIZE + 3; }
size_t ZBUFF_recommendedDOutSize() { return BLOCKSIZE; }

146
lib/zstd_buffered.h Normal file
View File

@ -0,0 +1,146 @@
/*
Buffered version of Zstd compression library
Copyright (C) 2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#ifndef ZSTD_BUFFERED_H
#define ZSTD_BUFFERED_H
/* The objects defined into this file should be considered experimental.
* They are not labelled stable, as their prototype may change in the future.
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
*/
#if defined (__cplusplus)
extern "C" {
#endif
/* *************************************
* Includes
***************************************/
#include <stddef.h> /* size_t */
/* *************************************
* Streaming functions
***************************************/
typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
ZBUFF_CCtx* ZBUFF_createCCtx(void);
size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
/** ************************************************
* Streaming compression
*
* A ZBUFF_CCtx object is required to track streaming operation.
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
* Use ZBUFF_compressInit() to start a new compression operation.
* ZBUFF_CCtx objects can be reused multiple times.
*
* Use ZBUFF_compressContinue() repetitively to consume input stream.
* *srcSizePtr and *maxDstSizePtr can be any size.
* The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr.
* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or move dst .
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
* or an error code, which can be tested using ZBUFF_isError().
*
* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
* Note that it will not output more than *maxDstSizePtr.
* Therefore, some content might still be left into its internal buffer if dst buffer is too small.
* @return : nb of bytes still present into internal buffer (0 if it's empty)
* or an error code, which can be tested using ZBUFF_isError().
*
* ZBUFF_compressEnd() instructs to finish a frame.
* It will perform a flush and write frame epilogue.
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small.
* In which case, call again ZBUFF_compressFlush() to complete the flush.
* @return : nb of bytes still present into internal buffer (0 if it's empty)
* or an error code, which can be tested using ZBUFF_isError().
*
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedCInSize / ZBUFF_recommendedCOutSize
* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, it improves latency to use this value.
* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering.
* By using both, you ensure that input will be entirely consumed, and output will always contain the result.
* **************************************************/
typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
ZBUFF_DCtx* ZBUFF_createDCtx(void);
size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
/** ************************************************
* Streaming decompression
*
* A ZBUFF_DCtx object is required to track streaming operation.
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
* Use ZBUFF_decompressInit() to start a new decompression operation.
* ZBUFF_DCtx objects can be reused multiple times.
*
* Use ZBUFF_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *maxDstSizePtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
* or 0 when a frame is completely decoded
* or an error code, which can be tested using ZBUFF_isError().
*
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize
* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* **************************************************/
/* *************************************
* Tool functions
***************************************/
unsigned ZBUFF_isError(size_t errorCode);
const char* ZBUFF_getErrorName(size_t errorCode);
/** The below functions provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are not compulsory, they just tend to offer better latency */
size_t ZBUFF_recommendedCInSize(void);
size_t ZBUFF_recommendedCOutSize(void);
size_t ZBUFF_recommendedDInSize(void);
size_t ZBUFF_recommendedDOutSize(void);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_BUFFERED_H */

62
lib/zstd_buffered_static.h Normal file
View File

@ -0,0 +1,62 @@
/*
zstd - buffered version of compression library
experimental complementary API, for static linking only
Copyright (C) 2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#ifndef ZSTD_BUFFERED_STATIC_H
#define ZSTD_BUFFERED_STATIC_H
/* The objects defined into this file should be considered experimental.
* They are not labelled stable, as their prototype may change in the future.
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
*/
#if defined (__cplusplus)
extern "C" {
#endif
/* *************************************
* Includes
***************************************/
#include "zstd_static.h"
#include "zstd_buffered.h"
/* *************************************
* Advanced Streaming functions
***************************************/
size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, ZSTD_parameters params);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_BUFFERED_STATIC_H */

70
lib/zstd_compress.c
View File

@ -138,7 +138,7 @@ static unsigned ZSTD_highbit(U32 val);
/** ZSTD_validateParams /** ZSTD_validateParams
correct params value to remain within authorized range correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */ optimize for srcSize if srcSize > 0 */
void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint) void ZSTD_validateParams(ZSTD_parameters* params)
{ {
const U32 btPlus = (params->strategy == ZSTD_btlazy2); const U32 btPlus = (params->strategy == ZSTD_btlazy2);
@ -147,12 +147,13 @@ void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint)
if (params->windowLog < ZSTD_WINDOWLOG_MIN) params->windowLog = ZSTD_WINDOWLOG_MIN; if (params->windowLog < ZSTD_WINDOWLOG_MIN) params->windowLog = ZSTD_WINDOWLOG_MIN;
/* correct params, to use less memory */ /* correct params, to use less memory */
if ((srcSizeHint > 0) && (srcSizeHint < (1<<ZSTD_WINDOWLOG_MAX))) if ((params->srcSize > 0) && (params->srcSize < (1<<ZSTD_WINDOWLOG_MAX)))
{ {
U32 srcLog = ZSTD_highbit((U32)srcSizeHint-1) + 1; U32 srcLog = ZSTD_highbit((U32)(params->srcSize)-1) + 1;
if (params->windowLog > srcLog) params->windowLog = srcLog; if (params->windowLog > srcLog) params->windowLog = srcLog;
} }
if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
if (params->contentLog > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus; /* <= ZSTD_CONTENTLOG_MAX */ if (params->contentLog > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus; /* <= ZSTD_CONTENTLOG_MAX */
if (params->contentLog < ZSTD_CONTENTLOG_MIN) params->contentLog = ZSTD_CONTENTLOG_MIN; if (params->contentLog < ZSTD_CONTENTLOG_MIN) params->contentLog = ZSTD_CONTENTLOG_MIN;
if (params->hashLog > ZSTD_HASHLOG_MAX) params->hashLog = ZSTD_HASHLOG_MAX; if (params->hashLog > ZSTD_HASHLOG_MAX) params->hashLog = ZSTD_HASHLOG_MAX;
@ -166,11 +167,8 @@ void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint)
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
ZSTD_parameters params, ZSTD_parameters params)
U64 srcSizeHint)
{ {
ZSTD_validateParams(&params, srcSizeHint);
/* reserve table memory */ /* reserve table memory */
{ {
const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog; const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
@ -207,10 +205,12 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
} }
/** ZSTD_reduceIndex
* rescale indexes to avoid future overflow (indexes are U32) */
static void ZSTD_reduceIndex (ZSTD_CCtx* zc, static void ZSTD_reduceIndex (ZSTD_CCtx* zc,
const U32 reducerValue) const U32 reducerValue)
{ {
const U32 contentLog = zc->params.strategy == ZSTD_fast ? 1 : zc->params.contentLog; const U32 contentLog = (zc->params.strategy == ZSTD_fast) ? 1 : zc->params.contentLog;
const U32 tableSpaceU32 = (1 << contentLog) + (1 << zc->params.hashLog); const U32 tableSpaceU32 = (1 << contentLog) + (1 << zc->params.hashLog);
U32* table32 = zc->hashTable; U32* table32 = zc->hashTable;
U32 index; U32 index;
@ -2038,30 +2038,51 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
} }
/** ZSTD_compressBegin_advanced
* Write frame header, according to params
* @return : nb of bytes written */
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize, void* dst, size_t maxDstSize,
const ZSTD_parameters params, ZSTD_parameters params)
const U64 srcSizeHint)
{ {
size_t errorCode; size_t errorCode;
if (maxDstSize < 4) return ERROR(dstSize_tooSmall);
errorCode = ZSTD_resetCCtx_advanced(ctx, params, srcSizeHint); ZSTD_validateParams(&params);
if (maxDstSize < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
errorCode = ZSTD_resetCCtx_advanced(ctx, params);
if (ZSTD_isError(errorCode)) return errorCode; if (ZSTD_isError(errorCode)) return errorCode;
MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */ MEM_writeLE32(dst, ZSTD_MAGICNUMBER); /* Write Header */
return 4; ((BYTE*)dst)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN);
return ZSTD_frameHeaderSize_min;
}
/** ZSTD_getParams
* return ZSTD_parameters structure for a selected compression level and srcSize.
* srcSizeHint value is optional, select 0 if not known */
ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint)
{
ZSTD_parameters result;
int tableID = ((srcSizeHint-1) <= 128 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
result = ZSTD_defaultParameters[tableID][compressionLevel];
result.srcSize = srcSizeHint;
return result;
} }
size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint) size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint)
{ {
int tableID = ((srcSizeHint-1) > 128 KB); /* intentional underflow for srcSizeHint == 0 */ return ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_getParams(compressionLevel, srcSizeHint));
if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
return ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_defaultParameters[tableID][compressionLevel], srcSizeHint);
} }
/** ZSTD_compressEnd
* Write frame epilogue
* @return : nb of bytes written into dst (or an error code) */
size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize) size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize)
{ {
BYTE* op = (BYTE*)dst; BYTE* op = (BYTE*)dst;
@ -2079,16 +2100,16 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize)
} }
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize, void* dst, size_t maxDstSize,
const void* src, size_t srcSize, const void* src, size_t srcSize,
ZSTD_parameters params) ZSTD_parameters params)
{ {
BYTE* const ostart = (BYTE*)dst; BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart; BYTE* op = ostart;
size_t oSize; size_t oSize;
/* Header */ /* Header */
oSize = ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, params, srcSize); oSize = ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, params);
if(ZSTD_isError(oSize)) return oSize; if(ZSTD_isError(oSize)) return oSize;
op += oSize; op += oSize;
maxDstSize -= oSize; maxDstSize -= oSize;
@ -2110,10 +2131,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel) size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
{ {
const int tableID = (srcSize > 128 KB); return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_getParams(compressionLevel, srcSize));
if (compressionLevel < 1) compressionLevel = 1;
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_defaultParameters[tableID][compressionLevel]);
} }
size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel) size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)

200
lib/zstd_decompress.c
View File

@ -120,6 +120,9 @@ const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
/* ************************************************************* /* *************************************************************
* Context management * Context management
***************************************************************/ ***************************************************************/
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage;
struct ZSTD_DCtx_s struct ZSTD_DCtx_s
{ {
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
@ -130,18 +133,21 @@ struct ZSTD_DCtx_s
void* vBase; void* vBase;
void* dictEnd; void* dictEnd;
size_t expected; size_t expected;
size_t headerSize;
ZSTD_parameters params;
blockType_t bType; blockType_t bType;
U32 phase; ZSTD_dStage stage;
const BYTE* litPtr; const BYTE* litPtr;
size_t litBufSize; size_t litBufSize;
size_t litSize; size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */]; BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */ }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx) size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
{ {
dctx->expected = ZSTD_frameHeaderSize; dctx->expected = ZSTD_frameHeaderSize_min;
dctx->phase = 0; dctx->stage = ZSTDds_getFrameHeaderSize;
dctx->previousDstEnd = NULL; dctx->previousDstEnd = NULL;
dctx->base = NULL; dctx->base = NULL;
dctx->vBase = NULL; dctx->vBase = NULL;
@ -167,6 +173,45 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
/* ************************************************************* /* *************************************************************
* Decompression section * Decompression section
***************************************************************/ ***************************************************************/
/** ZSTD_decodeFrameHeader_Part1
* decode the 1st part of the Frame Header, which tells Frame Header size.
* srcSize must be == ZSTD_frameHeaderSize_min
* @return : the full size of the Frame Header */
static size_t ZSTD_decodeFrameHeader_Part1(ZSTD_DCtx* zc, const void* src, size_t srcSize)
{
U32 magicNumber;
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
magicNumber = MEM_readLE32(src);
if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
zc->headerSize = ZSTD_frameHeaderSize_min;
return zc->headerSize;
}
/** ZSTD_decodeFrameHeader_Part2
* decode the full Frame Header
* srcSize must be the size provided by ZSTD_decodeFrameHeader_Part1
* @return : 0, or an error code, which can be tested using ZSTD_isError() */
static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
if (srcSize != zc->headerSize) return ERROR(srcSize_wrong);
memset(&(zc->params), 0, sizeof(zc->params));
zc->params.windowLog = ip[4] + ZSTD_WINDOWLOG_ABSOLUTEMIN;
return 0;
}
size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize)
{
U32 magicNumber;
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_max;
magicNumber = MEM_readLE32(src);
if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
memset(params, 0, sizeof(*params));
params->windowLog = ((const BYTE*)src)[4] + ZSTD_WINDOWLOG_ABSOLUTEMIN;
return 0;
}
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
{ {
@ -655,7 +700,6 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const v
BYTE* op = ostart; BYTE* op = ostart;
BYTE* const oend = ostart + maxDstSize; BYTE* const oend = ostart + maxDstSize;
size_t remainingSize = srcSize; size_t remainingSize = srcSize;
U32 magicNumber;
blockProperties_t blockProperties; blockProperties_t blockProperties;
@ -663,14 +707,23 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const v
ctx->base = ctx->vBase = ctx->dictEnd = dst; ctx->base = ctx->vBase = ctx->dictEnd = dst;
/* Frame Header */ /* Frame Header */
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); {
magicNumber = MEM_readLE32(src); size_t frameHeaderSize;
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(magicNumber)) {
return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber); const U32 magicNumber = MEM_readLE32(src);
if (ZSTD_isLegacy(magicNumber))
return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber);
}
#endif #endif
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); frameHeaderSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min);
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize; if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
frameHeaderSize = ZSTD_decodeFrameHeader_Part2(ctx, src, frameHeaderSize);
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
}
/* Loop on each block */ /* Loop on each block */
while (1) while (1)
@ -722,7 +775,6 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
/* ****************************** /* ******************************
* Streaming Decompression API * Streaming Decompression API
********************************/ ********************************/
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
{ {
return dctx->expected; return dctx->expected;
@ -732,7 +784,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con
{ {
/* Sanity check */ /* Sanity check */
if (srcSize != ctx->expected) return ERROR(srcSize_wrong); if (srcSize != ctx->expected) return ERROR(srcSize_wrong);
if (dst != ctx->previousDstEnd) /* not contiguous */ if (dst != ctx->previousDstEnd) /* not contiguous */
{ {
ctx->dictEnd = ctx->previousDstEnd; ctx->dictEnd = ctx->previousDstEnd;
if ((dst > ctx->base) && (dst < ctx->previousDstEnd)) /* rolling buffer : new segment right into tracked memory */ if ((dst > ctx->base) && (dst < ctx->previousDstEnd)) /* rolling buffer : new segment right into tracked memory */
@ -741,65 +793,85 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con
ctx->base = dst; ctx->base = dst;
} }
/* Decompress : frame header */ /* Decompress : frame header; part 1 */
if (ctx->phase == 0) switch (ctx->stage)
{ {
/* Check frame magic header */ case ZSTDds_getFrameHeaderSize :
U32 magicNumber = MEM_readLE32(src);
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
ctx->phase = 1;
ctx->expected = ZSTD_blockHeaderSize;
return 0;
}
/* Decompress : block header */
if (ctx->phase == 1)
{
blockProperties_t bp;
size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
if (ZSTD_isError(blockSize)) return blockSize;
if (bp.blockType == bt_end)
{ {
ctx->expected = 0; /* get frame header size */
ctx->phase = 0; if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
ctx->headerSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min);
if (ZSTD_isError(ctx->headerSize)) return ctx->headerSize;
memcpy(ctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
if (ctx->headerSize > ZSTD_frameHeaderSize_min)
{
ctx->expected = ctx->headerSize - ZSTD_frameHeaderSize_min;
ctx->stage = ZSTDds_decodeFrameHeader;
return 0;
}
ctx->expected = 0; /* not necessary to copy more */
} }
else case ZSTDds_decodeFrameHeader:
{ {
ctx->expected = blockSize; /* get frame header */
ctx->bType = bp.blockType; size_t result;
ctx->phase = 2; memcpy(ctx->headerBuffer + ZSTD_frameHeaderSize_min, src, ctx->expected);
result = ZSTD_decodeFrameHeader_Part2(ctx, ctx->headerBuffer, ctx->headerSize);
if (ZSTD_isError(result)) return result;
ctx->expected = ZSTD_blockHeaderSize;
ctx->stage = ZSTDds_decodeBlockHeader;
return 0;
} }
case ZSTDds_decodeBlockHeader:
ctx->previousDstEnd = dst;
return 0;
}
/* Decompress : block content */
{
size_t rSize;
switch(ctx->bType)
{ {
case bt_compressed: /* Decode block header */
rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize); blockProperties_t bp;
break; size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
case bt_raw : if (ZSTD_isError(blockSize)) return blockSize;
rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize); if (bp.blockType == bt_end)
break; {
case bt_rle : ctx->expected = 0;
return ERROR(GENERIC); /* not yet handled */ ctx->stage = ZSTDds_getFrameHeaderSize;
break; }
case bt_end : /* should never happen (filtered at phase 1) */ else
rSize = 0; {
break; ctx->expected = blockSize;
default: ctx->bType = bp.blockType;
return ERROR(GENERIC); ctx->stage = ZSTDds_decompressBlock;
} }
ctx->phase = 1;
ctx->expected = ZSTD_blockHeaderSize;
ctx->previousDstEnd = (char*)dst + rSize;
return rSize;
}
ctx->previousDstEnd = dst;
return 0;
}
case 3:
{
/* Decompress : block content */
size_t rSize;
switch(ctx->bType)
{
case bt_compressed:
rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize);
break;
case bt_raw :
rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize);
break;
case bt_rle :
return ERROR(GENERIC); /* not yet handled */
break;
case bt_end : /* should never happen (filtered at phase 1) */
rSize = 0;
break;
default:
return ERROR(GENERIC);
}
ctx->stage = ZSTDds_decodeBlockHeader;
ctx->expected = ZSTD_blockHeaderSize;
ctx->previousDstEnd = (char*)dst + rSize;
return rSize;
}
default:
return ERROR(GENERIC); /* impossible */
}
} }

5
lib/zstd_internal.h
View File

@ -54,6 +54,8 @@ extern "C" {
/* ************************************* /* *************************************
* Common constants * Common constants
***************************************/ ***************************************/
#define ZSTD_MAGICNUMBER 0xFD2FB524 /* v0.4 */
#define KB *(1 <<10) #define KB *(1 <<10)
#define MB *(1 <<20) #define MB *(1 <<20)
#define GB *(1U<<30) #define GB *(1U<<30)
@ -61,7 +63,8 @@ extern "C" {
#define BLOCKSIZE (128 KB) /* define, for static allocation */ #define BLOCKSIZE (128 KB) /* define, for static allocation */
static const size_t ZSTD_blockHeaderSize = 3; static const size_t ZSTD_blockHeaderSize = 3;
static const size_t ZSTD_frameHeaderSize = 4; static const size_t ZSTD_frameHeaderSize_min = 5;
#define ZSTD_frameHeaderSize_max 5 /* define, for static allocation */
#define BIT7 128 #define BIT7 128
#define BIT6 64 #define BIT6 64

241
lib/zstd_static.h
View File

@ -52,71 +52,9 @@ extern "C" {
/* ************************************* /* *************************************
* Types * Types
***************************************/ ***************************************/
/** from faster to stronger */
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
typedef struct
{
U32 windowLog; /* largest match distance : impact decompression buffer size */
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == more memory, faster*/
U32 searchLog; /* nb of searches : larger == more compression, slower*/
U32 searchLength; /* size of matches : larger == faster decompression */
ZSTD_strategy strategy;
} ZSTD_parameters;
/* *************************************
* Advanced function
***************************************/
/** ZSTD_compress_advanced
* Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
ZSTD_parameters params);
/** ZSTD_validateParams
correct params value to remain within authorized range
srcSizeHint value is optional, select 0 if not known */
void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint);
/* *************************************
* Streaming functions
***************************************/
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTD_DCtx* ZSTD_createDCtx(void);
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
/*
Use above functions alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.
Result is the number of bytes regenerated within 'dst'.
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
*/
/* *************************************
* Prefix - version detection
***************************************/
#define ZSTD_magicNumber 0xFD2FB523 /* v0.3 (current)*/
/* *************************************
* Pre-defined compression levels
***************************************/
#define ZSTD_MAX_CLEVEL 20
#define ZSTD_WINDOWLOG_MAX 26 #define ZSTD_WINDOWLOG_MAX 26
#define ZSTD_WINDOWLOG_MIN 18 #define ZSTD_WINDOWLOG_MIN 18
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11
#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1) #define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CONTENTLOG_MIN 4 #define ZSTD_CONTENTLOG_MIN 4
#define ZSTD_HASHLOG_MAX 28 #define ZSTD_HASHLOG_MAX 28
@ -126,55 +64,140 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
#define ZSTD_SEARCHLENGTH_MAX 7 #define ZSTD_SEARCHLENGTH_MAX 7
#define ZSTD_SEARCHLENGTH_MIN 4 #define ZSTD_SEARCHLENGTH_MIN 4
/** from faster to stronger */
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
typedef struct
{
U64 srcSize; /* optional : tells how much bytes are present in the frame. Use 0 if not known. */
U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == more memory, faster */
U32 searchLog; /* nb of searches : larger == more compression, slower */
U32 searchLength; /* size of matches : larger == faster decompression, sometimes less compression */
ZSTD_strategy strategy;
} ZSTD_parameters;
/* *************************************
* Advanced function
***************************************/
/** ZSTD_getParams
* return ZSTD_parameters structure for a selected compression level and srcSize.
* srcSizeHint value is optional, select 0 if not known */
ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint);
/** ZSTD_validateParams
* correct params value to remain within authorized range */
void ZSTD_validateParams(ZSTD_parameters* params);
/** ZSTD_compress_advanced
* Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
ZSTD_parameters params);
/* **************************************
* Streaming functions (bufferless mode)
****************************************/
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params);
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTD_DCtx* ZSTD_createDCtx(void);
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
/**
Streaming decompression, bufferless mode
A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times. Use ZSTD_resetDCtx() to return to fresh status.
First operation is to retrieve frame parameters, using ZSTD_getFrameParams().
This function doesn't consume its input. It needs enough input data to properly decode the frame header.
The objective is to retrieve *params.windowlog, to know how much memory is required during decoding.
Result : 0 if successfull, it means the ZSTD_parameters structure has been filled.
>0 : means there is not enough data into src. Provides the expected size to successfully decode header.
errorCode, which can be tested using ZSTD_isError() (For example, if it's not a ZSTD header)
Then it's possible to start decompression.
Use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() will use previous data blocks during decompress.
They should be located contiguously prior to current block. Alternatively, a round buffer is possible.
Just make sure that the combined of current and accessible past blocks is a minimum of (1 << windowlog).
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst'.
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
*/
/* *************************************
* Pre-defined compression levels
***************************************/
#define ZSTD_MAX_CLEVEL 20
static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = { static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = {
{ /* for <= 128 KB */ { /* "default" */
/* W, C, H, S, L, strat */ /* W, C, H, S, L, strat */
{ 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ { 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
{ 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */ { 0, 19, 13, 14, 1, 7, ZSTD_fast }, /* level 1 */
{ 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */ { 0, 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */
{ 17, 16, 17, 1, 5, ZSTD_fast }, /* level 3 */ { 0, 20, 18, 20, 1, 6, ZSTD_fast }, /* level 3 */
{ 17, 13, 15, 2, 4, ZSTD_greedy }, /* level 4 */ { 0, 21, 19, 21, 1, 6, ZSTD_fast }, /* level 4 */
{ 17, 15, 17, 3, 4, ZSTD_greedy }, /* level 5 */ { 0, 20, 14, 18, 3, 5, ZSTD_greedy }, /* level 5 */
{ 17, 14, 17, 3, 4, ZSTD_lazy }, /* level 6 */ { 0, 20, 18, 19, 3, 5, ZSTD_greedy }, /* level 6 */
{ 17, 16, 17, 4, 4, ZSTD_lazy }, /* level 7 */ { 0, 21, 17, 20, 3, 5, ZSTD_lazy }, /* level 7 */
{ 17, 16, 17, 4, 4, ZSTD_lazy2 }, /* level 8 */ { 0, 21, 19, 20, 3, 5, ZSTD_lazy }, /* level 8 */
{ 17, 17, 16, 5, 4, ZSTD_lazy2 }, /* level 9 */ { 0, 21, 20, 20, 3, 5, ZSTD_lazy2 }, /* level 9 */
{ 17, 17, 16, 6, 4, ZSTD_lazy2 }, /* level 10 */ { 0, 21, 19, 21, 4, 5, ZSTD_lazy2 }, /* level 10 */
{ 17, 17, 16, 7, 4, ZSTD_lazy2 }, /* level 11 */ { 0, 22, 20, 22, 4, 5, ZSTD_lazy2 }, /* level 11 */
{ 17, 17, 16, 8, 4, ZSTD_lazy2 }, /* level 12 */ { 0, 22, 20, 22, 5, 5, ZSTD_lazy2 }, /* level 12 */
{ 17, 18, 16, 4, 4, ZSTD_btlazy2 }, /* level 13 */ { 0, 22, 21, 22, 5, 5, ZSTD_lazy2 }, /* level 13 */
{ 17, 18, 16, 5, 4, ZSTD_btlazy2 }, /* level 14 */ { 0, 22, 22, 23, 5, 5, ZSTD_lazy2 }, /* level 14 */
{ 17, 18, 16, 6, 4, ZSTD_btlazy2 }, /* level 15 */ { 0, 23, 23, 23, 5, 5, ZSTD_lazy2 }, /* level 15 */
{ 17, 18, 16, 7, 4, ZSTD_btlazy2 }, /* level 16 */ { 0, 23, 21, 22, 5, 5, ZSTD_btlazy2 }, /* level 16 */
{ 17, 18, 16, 8, 4, ZSTD_btlazy2 }, /* level 17 */ { 0, 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */
{ 17, 18, 16, 9, 4, ZSTD_btlazy2 }, /* level 18 */ { 0, 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */
{ 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */ { 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */
{ 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */ { 0, 25, 26, 25, 6, 5, ZSTD_btlazy2 }, /* level 20 */
}, },
{ /* for > 128 KB */ { /* for srcSize <= 128 KB */
/* W, C, H, S, L, strat */ /* W, C, H, S, L, strat */
{ 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ { 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
{ 19, 13, 14, 1, 7, ZSTD_fast }, /* level 1 */ { 0, 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */
{ 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */ { 0, 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */
{ 20, 18, 20, 1, 6, ZSTD_fast }, /* level 3 */ { 0, 17, 16, 17, 1, 5, ZSTD_fast }, /* level 3 */
{ 21, 19, 21, 1, 6, ZSTD_fast }, /* level 4 */ { 0, 17, 13, 15, 2, 4, ZSTD_greedy }, /* level 4 */
{ 20, 14, 18, 3, 5, ZSTD_greedy }, /* level 5 */ { 0, 17, 15, 17, 3, 4, ZSTD_greedy }, /* level 5 */
{ 20, 18, 19, 3, 5, ZSTD_greedy }, /* level 6 */ { 0, 17, 14, 17, 3, 4, ZSTD_lazy }, /* level 6 */
{ 21, 17, 20, 3, 5, ZSTD_lazy }, /* level 7 */ { 0, 17, 16, 17, 4, 4, ZSTD_lazy }, /* level 7 */
{ 21, 19, 20, 3, 5, ZSTD_lazy }, /* level 8 */ { 0, 17, 16, 17, 4, 4, ZSTD_lazy2 }, /* level 8 */
{ 21, 20, 20, 3, 5, ZSTD_lazy2 }, /* level 9 */ { 0, 17, 17, 16, 5, 4, ZSTD_lazy2 }, /* level 9 */
{ 21, 19, 21, 4, 5, ZSTD_lazy2 }, /* level 10 */ { 0, 17, 17, 16, 6, 4, ZSTD_lazy2 }, /* level 10 */
{ 22, 20, 22, 4, 5, ZSTD_lazy2 }, /* level 11 */ { 0, 17, 17, 16, 7, 4, ZSTD_lazy2 }, /* level 11 */
{ 22, 20, 22, 5, 5, ZSTD_lazy2 }, /* level 12 */ { 0, 17, 17, 16, 8, 4, ZSTD_lazy2 }, /* level 12 */
{ 22, 21, 22, 5, 5, ZSTD_lazy2 }, /* level 13 */ { 0, 17, 18, 16, 4, 4, ZSTD_btlazy2 }, /* level 13 */
{ 22, 22, 23, 5, 5, ZSTD_lazy2 }, /* level 14 */ { 0, 17, 18, 16, 5, 4, ZSTD_btlazy2 }, /* level 14 */
{ 23, 23, 23, 5, 5, ZSTD_lazy2 }, /* level 15 */ { 0, 17, 18, 16, 6, 4, ZSTD_btlazy2 }, /* level 15 */
{ 23, 21, 22, 5, 5, ZSTD_btlazy2 }, /* level 16 */ { 0, 17, 18, 16, 7, 4, ZSTD_btlazy2 }, /* level 16 */
{ 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */ { 0, 17, 18, 16, 8, 4, ZSTD_btlazy2 }, /* level 17 */
{ 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */ { 0, 17, 18, 16, 9, 4, ZSTD_btlazy2 }, /* level 18 */
{ 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */ { 0, 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */
{ 25, 26, 25, 6, 5, ZSTD_btlazy2 }, /* level 20 */ { 0, 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */
} },
}; };

4
programs/Makefile
View File

@ -58,12 +58,12 @@ default: zstd
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen
zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \ zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \ $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
zstd32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \ zstd32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \ $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)

284
programs/fileio.c
View File

@ -49,8 +49,6 @@
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif #endif
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */ #define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */
#define _POSIX_SOURCE 1 /* enable fileno() within <stdio.h> on unix */ #define _POSIX_SOURCE 1 /* enable fileno() within <stdio.h> on unix */
@ -67,10 +65,11 @@
#include <sys/stat.h> /* stat64 */ #include <sys/stat.h> /* stat64 */
#include "mem.h" #include "mem.h"
#include "fileio.h" #include "fileio.h"
#include "zstd_static.h" #include "zstd_static.h" /* ZSTD_magicNumber */
#include "zstd_buffered_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_legacy.h" /* legacy */ # include "zstd_legacy.h" /* legacy */
# include "fileio_legacy.h" /* legacy */ # include "fileio_legacy.h" /* legacy */
#endif #endif
@ -82,7 +81,7 @@
# include <fcntl.h> /* _O_BINARY */ # include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */ # include <io.h> /* _setmode, _isatty */
# ifdef __MINGW32__ # ifdef __MINGW32__
/* int _fileno(FILE *stream); // seems no longer useful // MINGW somehow forgets to include this windows declaration into <stdio.h> */ // int _fileno(FILE *stream); /* seems no longer useful /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
# endif # endif
# define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; } # define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
@ -114,8 +113,8 @@
#define BIT6 0x40 #define BIT6 0x40
#define BIT7 0x80 #define BIT7 0x80
//static const unsigned FIO_maxBlockSizeID = 0xB; /* => 2MB block */ #define BLOCKSIZE (128 KB)
static const unsigned FIO_blockHeaderSize = 3; #define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */ #define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
#define FSE_CHECKSUM_SEED 0 #define FSE_CHECKSUM_SEED 0
@ -241,36 +240,25 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
U64 filesize = 0; U64 filesize = 0;
U64 compressedfilesize = 0; U64 compressedfilesize = 0;
BYTE* inBuff; BYTE* inBuff;
BYTE* inSlot;
BYTE* inEnd;
BYTE* outBuff; BYTE* outBuff;
size_t blockSize = 128 KB; size_t inBuffSize = ZBUFF_recommendedCInSize();
size_t inBuffSize = 4 * blockSize; size_t outBuffSize = ZBUFF_recommendedCOutSize();
size_t outBuffSize = ZSTD_compressBound(blockSize);
FILE* finput; FILE* finput;
FILE* foutput; FILE* foutput;
size_t sizeCheck, cSize; size_t sizeCheck, errorCode;
ZSTD_CCtx* ctx; ZBUFF_CCtx* ctx;
/* Allocate Memory */
ctx = ZBUFF_createCCtx();
inBuff = (BYTE*)malloc(inBuffSize);
outBuff = (BYTE*)malloc(outBuffSize);
if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory");
/* init */ /* init */
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
filesize = FIO_getFileSize(input_filename); filesize = FIO_getFileSize(input_filename);
errorCode = ZBUFF_compressInit_advanced(ctx, ZSTD_getParams(cLevel, filesize));
/* Allocate Memory */ if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing compression");
ctx = ZSTD_createCCtx();
inBuff = (BYTE*)malloc(inBuffSize);
outBuff = (BYTE*)malloc(outBuffSize);
if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory");
inSlot = inBuff;
inEnd = inBuff + inBuffSize;
/* Write Frame Header */
cSize = ZSTD_compressBegin(ctx, outBuff, outBuffSize, cLevel, filesize);
if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename);
compressedfilesize += cSize;
filesize = 0; filesize = 0;
/* Main compression loop */ /* Main compression loop */
@ -279,33 +267,41 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
size_t inSize; size_t inSize;
/* Fill input Buffer */ /* Fill input Buffer */
if (inSlot + blockSize > inEnd) inSlot = inBuff; inSize = fread(inBuff, (size_t)1, inBuffSize, finput);
inSize = fread(inSlot, (size_t)1, blockSize, finput);
if (inSize==0) break; if (inSize==0) break;
filesize += inSize; filesize += inSize;
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20)); DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
/* Compress Block */ {
cSize = ZSTD_compressContinue(ctx, outBuff, outBuffSize, inSlot, inSize); /* Compress (buffered streaming ensures appropriate formatting) */
if (ZSTD_isError(cSize)) size_t usedInSize = inSize;
EXM_THROW(24, "Compression error : %s ", ZSTD_getErrorName(cSize)); size_t cSize = outBuffSize;
size_t result = ZBUFF_compressContinue(ctx, outBuff, &cSize, inBuff, &usedInSize);
if (ZBUFF_isError(result))
EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result));
if (inSize != usedInSize)
/* inBuff should be entirely consumed since buffer sizes are recommended ones */
EXM_THROW(24, "Compression error : input block not fully consumed");
/* Write cBlock */ /* Write cBlock */
sizeCheck = fwrite(outBuff, 1, cSize, foutput); sizeCheck = fwrite(outBuff, 1, cSize, foutput);
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename); if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
compressedfilesize += cSize; compressedfilesize += cSize;
inSlot += inSize; }
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100); DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
} }
/* End of Frame */ /* End of Frame */
cSize = ZSTD_compressEnd(ctx, outBuff, outBuffSize); {
if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end"); size_t cSize = outBuffSize;
size_t result = ZBUFF_compressEnd(ctx, outBuff, &cSize);
if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
sizeCheck = fwrite(outBuff, 1, cSize, foutput); sizeCheck = fwrite(outBuff, 1, cSize, foutput);
if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename); if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
compressedfilesize += cSize; compressedfilesize += cSize;
}
/* Status */ /* Status */
DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "\r%79s\r", "");
@ -315,7 +311,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
/* clean */ /* clean */
free(inBuff); free(inBuff);
free(outBuff); free(outBuff);
ZSTD_freeCCtx(ctx); ZBUFF_freeCCtx(ctx);
fclose(finput); fclose(finput);
if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename); if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
@ -324,124 +320,87 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput, unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
BYTE* inBuff, size_t inBuffSize, BYTE* inBuff, size_t inBuffSize, size_t alreadyLoaded,
BYTE* outBuff, size_t outBuffSize, BYTE* outBuff, size_t outBuffSize,
ZSTD_DCtx* dctx) ZBUFF_DCtx* dctx)
{ {
BYTE* op = outBuff; U64 frameSize = 0;
BYTE* const oend = outBuff + outBuffSize; size_t readSize=alreadyLoaded;
U64 filesize = 0;
size_t toRead;
size_t sizeCheck;
/* Main decompression Loop */ /* Main decompression Loop */
toRead = ZSTD_nextSrcSizeToDecompress(dctx); ZBUFF_decompressInit(dctx);
while (toRead) while (1)
{ {
size_t readSize, decodedSize; /* Decode */
size_t sizeCheck;
size_t inSize=readSize, decodedSize=outBuffSize;
size_t inStart=0;
size_t toRead = ZBUFF_decompressContinue(dctx, outBuff, &decodedSize, inBuff+inStart, &inSize);
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
if (toRead==0) break; /* end of Frame */
readSize -= inSize;
inStart += inSize;
/* Write block */
sizeCheck = fwrite(outBuff, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (readSize) continue; /* still some data left within inBuff */
/* Fill input buffer */ /* Fill input buffer */
if (toRead > inBuffSize) if (toRead > inBuffSize) EXM_THROW(34, "too large block");
EXM_THROW(34, "too large block");
readSize = fread(inBuff, 1, toRead, finput); readSize = fread(inBuff, 1, toRead, finput);
if (readSize != toRead) if (readSize != toRead) EXM_THROW(35, "Read error");
EXM_THROW(35, "Read error");
/* Decode block */
decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize);
if (ZSTD_isError(decodedSize)) EXM_THROW(36, "Decoding error : input corrupted");
if (decodedSize) /* not a header */
{
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
filesize += decodedSize;
op += decodedSize;
if (op==oend) op = outBuff;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) );
}
/* prepare for next Block */
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
} }
return filesize; return frameSize;
} }
#define MAXHEADERSIZE (FIO_FRAMEHEADERSIZE+3)
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename) unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
{ {
FILE* finput, *foutput; FILE* finput, *foutput;
BYTE* inBuff=NULL; BYTE* inBuff=NULL;
size_t inBuffSize = 0; size_t inBuffSize = ZBUFF_recommendedDInSize();
BYTE* outBuff=NULL; BYTE* outBuff=NULL;
size_t outBuffSize = 0; size_t outBuffSize = ZBUFF_recommendedDOutSize();
U32 blockSize = 128 KB;
U32 wNbBlocks = 4;
U64 filesize = 0; U64 filesize = 0;
BYTE* header[MAXHEADERSIZE];
size_t toRead; size_t toRead;
size_t sizeCheck; size_t sizeCheck;
/* Init */ /* Init */
ZSTD_DCtx* dctx = ZSTD_createDCtx(); ZBUFF_DCtx* dctx = ZBUFF_createDCtx();
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
/* Allocate Memory (if needed) */
inBuff = (BYTE*)malloc(inBuffSize);
outBuff = (BYTE*)malloc(outBuffSize);
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
/* for each frame */ /* for each frame */
for ( ; ; ) for ( ; ; )
{ {
/* check magic number -> version */
U32 magicNumber; U32 magicNumber;
toRead = sizeof(ZSTD_magicNumber);; toRead = 0;
sizeCheck = fread(header, (size_t)1, toRead, finput);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
/* check magic number -> version */
toRead = 4;
sizeCheck = fread(inBuff, (size_t)1, toRead, finput);
if (sizeCheck==0) break; /* no more input */ if (sizeCheck==0) break; /* no more input */
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header"); if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
magicNumber = MEM_readLE32(inBuff);
magicNumber = MEM_readLE32(header);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(magicNumber)) if (ZSTD_isLegacy(magicNumber))
{ {
filesize += FIO_decompressLegacyFrame(foutput, finput, magicNumber); filesize += FIO_decompressLegacyFrame(foutput, finput, magicNumber);
continue; continue;
} }
#endif /* ZSTD_LEGACY_SUPPORT */ #endif /* ZSTD_LEGACY_SUPPORT */
if (magicNumber != ZSTD_magicNumber) EXM_THROW(32, "Error : unknown frame prefix");
/* prepare frame decompression, by completing header */ filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, toRead, outBuff, outBuffSize, dctx);
ZSTD_resetDCtx(dctx);
toRead = ZSTD_nextSrcSizeToDecompress(dctx) - sizeof(ZSTD_magicNumber);
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
sizeCheck = fread(&header[sizeof(ZSTD_magicNumber)], 1, toRead, finput);
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, sizeof(ZSTD_magicNumber)+toRead); // Decode frame header
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
/* Here later : blockSize determination */
/* Allocate Memory (if needed) */
{
size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
size_t newOutBuffSize = wNbBlocks * blockSize;
if (newInBuffSize > inBuffSize)
{
free(inBuff);
inBuffSize = newInBuffSize;
inBuff = (BYTE*)malloc(inBuffSize);
}
if (newOutBuffSize > outBuffSize)
{
free(outBuff);
outBuffSize = newOutBuffSize;
outBuff = (BYTE*)malloc(outBuffSize);
}
}
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
} }
DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "\r%79s\r", "");
@ -450,7 +409,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
/* clean */ /* clean */
free(inBuff); free(inBuff);
free(outBuff); free(outBuff);
ZSTD_freeDCtx(dctx); ZBUFF_freeDCtx(dctx);
fclose(finput); fclose(finput);
if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename); if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
@ -458,74 +417,3 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
} }
#if 0
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
{
FILE* finput, *foutput;
BYTE* inBuff=NULL;
size_t inBuffSize = 0;
BYTE* outBuff=NULL;
size_t outBuffSize = 0;
U32 blockSize = 128 KB;
U32 wNbBlocks = 4;
U64 filesize = 0;
BYTE* header[MAXHEADERSIZE];
ZSTD_Dctx* dctx;
size_t toRead;
size_t sizeCheck;
/* Init */
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
dctx = ZSTD_createDCtx();
/* for each frame */
for ( ; ; )
{
/* check header */
ZSTD_resetDCtx(dctx);
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
sizeCheck = fread(header, (size_t)1, toRead, finput);
if (sizeCheck==0) break; /* no more input */
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
/* Here later : blockSize determination */
/* Allocate Memory (if needed) */
{
size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
size_t newOutBuffSize = wNbBlocks * blockSize;
if (newInBuffSize > inBuffSize)
{
free(inBuff);
inBuffSize = newInBuffSize;
inBuff = (BYTE*)malloc(inBuffSize);
}
if (newOutBuffSize > outBuffSize)
{
free(outBuff);
outBuffSize = newOutBuffSize;
outBuff = (BYTE*)malloc(outBuffSize);
}
}
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
}
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "Decoded %llu bytes \n", (long long unsigned)filesize);
/* clean */
free(inBuff);
free(outBuff);
ZSTD_freeDCtx(dctx);
fclose(finput);
if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
return filesize;
}
#endif

67
programs/fullbench.c
View File

@ -243,31 +243,6 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &length, DTableLL, DTableML, DTableOffb, buff2, g_cSize); return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &length, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
} }
size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
U32 i;
size_t total = 0;
BYTE* data = (BYTE*)buff2;
(void)dst; (void)dstSize; (void)src;
for (i=0; i < srcSize; i++)
{
U32 b = data[i];
total += b;
if (b==0) total = 0; // 825
//if (!b) total = 0; // 825
//total = b ? total : 0; // 622
//total &= -!b; // 622
//total *= !!b; // 465
}
return total;
}
size_t local_decodeLiteralsForward(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
(void)src; (void)srcSize;
return FSE_decompress(dst, dstSize, buff2, g_cSize);
}
@ -300,12 +275,6 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
case 32: case 32:
benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders"; benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
break; break;
case 101:
benchFunction = local_conditionalNull; benchName = "conditionalNull";
break;
case 102:
benchFunction = local_decodeLiteralsForward; benchName = "ZSTD_decodeLiteralsForward";
break;
default : default :
return 0; return 0;
} }
@ -332,14 +301,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
{ {
blockProperties_t bp; blockProperties_t bp;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block type ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) if (bp.blockType != bt_compressed)
{ {
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n"); DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
goto _cleanOut; goto _cleanOut;
} }
memcpy(buff2, dstBuff+7, g_cSize-7); memcpy(buff2, dstBuff+8, g_cSize-8);
srcSize = srcSize > 128 KB ? 128 KB : srcSize; // relative to block srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
break; break;
} }
case 32: /* ZSTD_decodeSeqHeaders */ case 32: /* ZSTD_decodeSeqHeaders */
@ -348,9 +317,9 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
const BYTE* ip = dstBuff; const BYTE* ip = dstBuff;
const BYTE* iend; const BYTE* iend;
size_t blockSize; size_t blockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
ip += 4; // Jump magic Number ip += 5; /* Skip frame Header */
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block type blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) if (bp.blockType != bt_compressed)
{ {
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
@ -358,32 +327,16 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
} }
iend = ip + 3 + blockSize; /* End of first block */ iend = ip + 3 + blockSize; /* End of first block */
ip += 3; /* skip block header */ ip += 3; /* skip block header */
ip += ZSTD_decodeLiteralsBlock(g_dctxPtr, ip, iend-ip); // jump literal sub block and its header ip += ZSTD_decodeLiteralsBlock(g_dctxPtr, ip, iend-ip); /* skip literal segment */
g_cSize = iend-ip; g_cSize = iend-ip;
memcpy(buff2, ip, g_cSize); // copy rest of block (starting with SeqHeader) memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */
srcSize = srcSize > 128 KB ? 128 KB : srcSize; // speed relative to block srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
break; break;
} }
/* test functions */ /* test functions */
/* by convention, test functions can be added > 100 */
case 101: /* conditionalNull */
{
size_t i;
for (i=0; i<srcSize; i++)
buff2[i] = i & 15;
break;
}
case 102: /* local_decodeLiteralsForward */
{
blockProperties_t bp;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp);
memcpy(buff2, dstBuff+10, g_cSize);
//srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
srcSize = srcSize > 128 KB ? 128 KB : srcSize; // relative to block
break;
}
default : ; default : ;
} }

50
programs/paramgrill.c
View File

@ -46,7 +46,7 @@
#if defined(_MSC_VER) #if defined(_MSC_VER)
# define snprintf _snprintf /* snprintf unsupported by Visual <= 2012 */ # define snprintf _snprintf /* snprintf unsupported by Visual <= 2012 */
#endif #endif
/************************************** /**************************************
* Includes * Includes
@ -125,8 +125,7 @@ static U32 g_rand = 1;
static U32 g_singleRun = 0; static U32 g_singleRun = 0;
static U32 g_target = 0; static U32 g_target = 0;
static U32 g_noSeed = 0; static U32 g_noSeed = 0;
static const ZSTD_parameters* g_seedParams = ZSTD_defaultParameters[0]; static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_greedy };
void BMK_SetNbIterations(int nbLoops) void BMK_SetNbIterations(int nbLoops)
{ {
@ -139,28 +138,6 @@ void BMK_SetNbIterations(int nbLoops)
* Private functions * Private functions
*********************************************************/ *********************************************************/
static unsigned BMK_highbit(U32 val)
{
# if defined(_MSC_VER) /* Visual */
unsigned long r;
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
return 31 - __builtin_clz(val);
# else /* Software version */
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
int r;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
return r;
# endif
}
#if defined(BMK_LEGACY_TIMER) #if defined(BMK_LEGACY_TIMER)
static int BMK_GetMilliStart(void) static int BMK_GetMilliStart(void)
@ -655,7 +632,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
/* validate new conf */ /* validate new conf */
{ {
ZSTD_parameters saved = p; ZSTD_parameters saved = p;
ZSTD_validateParams(&p, g_blockSize ? g_blockSize : srcSize); ZSTD_validateParams(&p);
if (memcmp(&p, &saved, sizeof(p))) continue; /* p was invalid */ if (memcmp(&p, &saved, sizeof(p))) continue; /* p was invalid */
} }
@ -707,12 +684,12 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
const char* rfName = "grillResults.txt"; const char* rfName = "grillResults.txt";
FILE* f; FILE* f;
const size_t blockSize = g_blockSize ? g_blockSize : srcSize; const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
const U32 srcLog = BMK_highbit((U32)(blockSize-1))+1;
if (g_singleRun) if (g_singleRun)
{ {
BMK_result_t testResult; BMK_result_t testResult;
ZSTD_validateParams(&g_params, blockSize); g_params.srcSize = blockSize;
ZSTD_validateParams(&g_params);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
DISPLAY("\n"); DISPLAY("\n");
return; return;
@ -735,9 +712,10 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
params.searchLog = 1; params.searchLog = 1;
params.searchLength = 7; params.searchLength = 7;
params.strategy = ZSTD_fast; params.strategy = ZSTD_fast;
ZSTD_validateParams(&params, blockSize); params.srcSize = blockSize;
ZSTD_validateParams(&params);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4; g_cSpeedTarget[1] = (testResult.cSpeed * 31) >> 5;
} }
/* establish speed objectives (relative to level 1) */ /* establish speed objectives (relative to level 1) */
@ -746,16 +724,10 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
/* populate initial solution */ /* populate initial solution */
{ {
const int tableID = (blockSize > 128 KB);
const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL; const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL;
g_seedParams = ZSTD_defaultParameters[tableID];
for (i=1; i<=maxSeeds; i++) for (i=1; i<=maxSeeds; i++)
{ {
const U32 btPlus = (params.strategy == ZSTD_btlazy2); params = ZSTD_getParams(i, blockSize);
params = g_seedParams[i];
params.windowLog = MIN(srcLog, params.windowLog);
params.contentLog = MIN(params.windowLog+btPlus, params.contentLog);
params.searchLog = MIN(params.contentLog, params.searchLog);
BMK_seed(winners, params, srcBuffer, srcSize, ctx); BMK_seed(winners, params, srcBuffer, srcSize, ctx);
} }
} }
@ -963,7 +935,7 @@ int main(int argc, char** argv)
case 'S': case 'S':
g_singleRun = 1; g_singleRun = 1;
argument++; argument++;
g_params = g_seedParams[2]; g_params = ZSTD_getParams(2, g_blockSize);
for ( ; ; ) for ( ; ; )
{ {
switch(*argument) switch(*argument)
@ -1013,7 +985,7 @@ int main(int argc, char** argv)
argument++; argument++;
while ((*argument>= '0') && (*argument<='9')) while ((*argument>= '0') && (*argument<='9'))
cLevel *= 10, cLevel += *argument++ - '0'; cLevel *= 10, cLevel += *argument++ - '0';
g_params = g_seedParams[cLevel]; g_params = ZSTD_getParams(cLevel, g_blockSize);
continue; continue;
} }
default : ; default : ;