krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-06 16:56:49 +02:00
Code Issues Releases 1 Activity

quick first zstdhc version

Browse Source
This commit is contained in:
Yann Collet 2015-10-22 15:31:46 +01:00
parent 353c5d26cf
commit f3eca25322
9 changed files with 866 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

215
lib/zstd.c
View File

@ -130,12 +130,6 @@ static const U32 g_searchStrength = 8;
#define WORKPLACESIZE (BLOCKSIZE*3)
#define MINMATCH 4
#define MLbits 7
#define LLbits 6
#define Offbits 5
#define MaxML ((1<<MLbits )-1)
#define MaxLL ((1<<LLbits )-1)
#define MaxOff 31
#define LitFSELog 11
#define MLFSELog 10
#define LLFSELog 10
@ -153,49 +147,12 @@ static const size_t ZSTD_frameHeaderSize = 4;
/* *******************************************************
* Memory operations
**********************************************************/
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
do COPY8(op, ip) while (op < oend);
}
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
/* **************************************
* Local structures
****************************************/
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
typedef struct
{
blockType_t blockType;
U32 origSize;
} blockProperties_t;
typedef struct {
void* buffer;
U32* offsetStart;
U32* offset;
BYTE* offCodeStart;
BYTE* offCode;
BYTE* litStart;
BYTE* lit;
BYTE* litLengthStart;
BYTE* litLength;
BYTE* matchLengthStart;
BYTE* matchLength;
BYTE* dumpsStart;
BYTE* dumps;
} seqStore_t;
void ZSTD_resetSeqStore(seqStore_t* ssPtr)
{
ssPtr->offset = ssPtr->offsetStart;
@ -205,7 +162,6 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr)
ssPtr->dumps = ssPtr->dumpsStart;
}
struct ZSTD_CCtx_s
{
const BYTE* base;
@ -231,7 +187,7 @@ void ZSTD_resetCCtx(ZSTD_CCtx* ctx)
ctx->seqStore.litLengthStart = ctx->seqStore.litStart + BLOCKSIZE;
ctx->seqStore.matchLengthStart = ctx->seqStore.litLengthStart + (BLOCKSIZE>>2);
ctx->seqStore.dumpsStart = ctx->seqStore.matchLengthStart + (BLOCKSIZE>>2);
memset(ctx->hashTable, 0, HASH_TABLESIZE*4);
memset(ctx->hashTable, 0, sizeof(ctx->hashTable));
}
ZSTD_CCtx* ZSTD_createCCtx(void)
@ -289,100 +245,6 @@ static unsigned ZSTD_highbit(U32 val)
}
/* *************************************
* Function body to include
***************************************/
#include "mem.h"
static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
{
if (MEM_isLittleEndian())
{
if (MEM_64bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
}
else /* Big Endian CPU */
{
if (MEM_32bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
r += (!val);
return r;
# endif
}
}
}
MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while ((pIn<pInLimit-(sizeof(size_t)-1)))
{
size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart);
}
if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (size_t)(pIn - pStart);
}
/* *******************************************************
* Compression
*********************************************************/
@ -392,7 +254,7 @@ size_t ZSTD_compressBound(size_t srcSize) /* maximum compressed size */
}
static size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
BYTE* const ostart = (BYTE* const)dst;
@ -433,7 +295,7 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t maxDstSize, const
size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 1; }
static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
const void* src, size_t srcSize)
{
const size_t minGain = ZSTD_minGain(srcSize);
BYTE* const ostart = (BYTE*)dst;
@ -461,9 +323,9 @@ static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
}
static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
const seqStore_t* seqStorePtr,
size_t srcSize)
size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
const seqStore_t* seqStorePtr,
size_t srcSize)
{
U32 count[MaxSeq+1];
S16 norm[MaxSeq+1];
@ -475,7 +337,6 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
U32 CTable_MatchLength[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML )];
U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
const BYTE* const op_lit_start = seqStorePtr->litStart;
const BYTE* op_lit = seqStorePtr->lit;
const BYTE* const llTable = seqStorePtr->litLengthStart;
const BYTE* const llPtr = seqStorePtr->litLength;
const BYTE* const mlTable = seqStorePtr->matchLengthStart;
@ -492,7 +353,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
/* Compress literals */
{
size_t cSize;
size_t litSize = op_lit - op_lit_start;
size_t litSize = seqStorePtr->lit - op_lit_start;
if (litSize <= LITERAL_NOENTROPY)
cSize = ZSTD_compressRawLiteralsBlock(op, maxDstSize, op_lit_start, litSize);
@ -665,48 +526,6 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
}
static void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
{
BYTE* op_lit = seqStorePtr->lit;
BYTE* const l_end = op_lit + litLength;
/* copy Literals */
while (op_lit<l_end) COPY8(op_lit, literals);
seqStorePtr->lit += litLength;
/* literal Length */
if (litLength >= MaxLL)
{
*(seqStorePtr->litLength++) = MaxLL;
if (litLength<255 + MaxLL)
*(seqStorePtr->dumps++) = (BYTE)(litLength - MaxLL);
else
{
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
}
}
else *(seqStorePtr->litLength++) = (BYTE)litLength;
/* match offset */
*(seqStorePtr->offset++) = (U32)offset;
/* match Length */
if (matchLength >= MaxML)
{
*(seqStorePtr->matchLength++) = MaxML;
if (matchLength < 255+MaxML)
*(seqStorePtr->dumps++) = (BYTE)(matchLength - MaxML);
else
{
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)matchLength); seqStorePtr->dumps+=3;
}
}
else *(seqStorePtr->matchLength++) = (BYTE)matchLength;
}
//static const U32 hashMask = (1<<HASH_LOG)-1;
//static const U64 prime5bytes = 889523592379ULL;
//static const U64 prime6bytes = 227718039650203ULL;
@ -751,9 +570,9 @@ static size_t ZSTD_compressBlock(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, c
const BYTE* ip = istart + 1;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 16;
const BYTE* const ilimit = iend - 8;
size_t prevOffset=0, offset=0;
size_t prevOffset=4, offset=4;
/* init */
@ -1095,7 +914,6 @@ size_t ZSTD_decodeLiteralsBlock(void* ctx,
switch(*istart & 3)
{
default:
case 0:
{
size_t nbLiterals = BLOCKSIZE;
@ -1124,6 +942,8 @@ size_t ZSTD_decodeLiteralsBlock(void* ctx,
dctx->litSize = litSize;
return 4;
}
default:
return ERROR(corruption_detected); /* forbidden nominal case */
}
}
@ -1343,8 +1163,8 @@ static size_t ZSTD_execSequence(BYTE* op,
const BYTE* match = op - sequence.offset;
/* check */
if (sequence.offset > (size_t)op) return ERROR(corruption_detected); /* address space overflow test (this test seems kept by clang optimizer) */
//if (match > op) return ERROR(corruption_detected); /* address space overflow test (is clang optimizer removing this test ?) */
if (sequence.offset > (size_t)op) return ERROR(corruption_detected); /* address space overflow test (this test seems kept by clang optimizer) */
if (match < base) return ERROR(corruption_detected);
/* close range match, overlap */
@ -1419,9 +1239,10 @@ static size_t ZSTD_decompressSequences(
seqState_t seqState;
memset(&sequence, 0, sizeof(sequence));
sequence.offset = 4;
seqState.dumps = dumps;
seqState.dumpsEnd = dumps + dumpsLength;
seqState.prevOffset = 1;
seqState.prevOffset = 4;
errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
if (ERR_isError(errorCode)) return ERROR(corruption_detected);
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
@ -1447,7 +1268,7 @@ static size_t ZSTD_decompressSequences(
size_t lastLLSize = litEnd - litPtr;
if (litPtr > litEnd) return ERROR(corruption_detected);
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
if (op != litPtr) memmove(op, litPtr, lastLLSize);
if (op != litPtr) memcpy(op, litPtr, lastLLSize);
op += lastLLSize;
}
}
@ -1542,9 +1363,9 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
}
/*******************************
/* ******************************
* Streaming Decompression API
*******************************/
********************************/
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
{

189
lib/zstd_static.h
View File

@ -82,6 +82,195 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
#include "error.h"
/* *************************************
* Function body to include
***************************************/
#include "mem.h"
static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
{
if (MEM_isLittleEndian())
{
if (MEM_64bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
}
else /* Big Endian CPU */
{
if (MEM_32bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
r += (!val);
return r;
# endif
}
}
}
MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while ((pIn<pInLimit-(sizeof(size_t)-1)))
{
size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart);
}
if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (size_t)(pIn - pStart);
}
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
do COPY8(op, ip) while (op < oend);
}
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
typedef struct
{
blockType_t blockType;
U32 origSize;
} blockProperties_t;
size_t ZSTD_noCompressBlock(void* op, size_t maxDstSize, const void* ip, size_t blockSize);
typedef struct {
void* buffer;
U32* offsetStart;
U32* offset;
BYTE* offCodeStart;
BYTE* offCode;
BYTE* litStart;
BYTE* lit;
BYTE* litLengthStart;
BYTE* litLength;
BYTE* matchLengthStart;
BYTE* matchLength;
BYTE* dumpsStart;
BYTE* dumps;
} seqStore_t;
void ZSTD_resetSeqStore(seqStore_t* ssPtr);
#define MLbits 7
#define LLbits 6
#define Offbits 5
#define MaxML ((1<<MLbits) - 1)
#define MaxLL ((1<<LLbits) - 1)
#define MaxOff 31
/** ZSTD_storeSeq
Store a sequence (literal length, literals, offset code and match length) into seqStore_t
@offsetCode : distance to match, or 0 == repCode
@matchCode : matchLength - MINMATCH
*/
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
{
/* copy Literals */
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
seqStorePtr->lit += litLength;
/* literal Length */
if (litLength >= MaxLL)
{
*(seqStorePtr->litLength++) = MaxLL;
if (litLength<255 + MaxLL)
*(seqStorePtr->dumps++) = (BYTE)(litLength - MaxLL);
else
{
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
}
}
else *(seqStorePtr->litLength++) = (BYTE)litLength;
/* match offset */
*(seqStorePtr->offset++) = (U32)offsetCode;
/* match Length */
if (matchCode >= MaxML)
{
*(seqStorePtr->matchLength++) = MaxML;
if (matchCode < 255+MaxML)
*(seqStorePtr->dumps++) = (BYTE)(matchCode - MaxML);
else
{
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)matchCode); seqStorePtr->dumps += 3;
}
}
else *(seqStorePtr->matchLength++) = (BYTE)matchCode;
}
size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, const seqStore_t* seqStorePtr, size_t srcSize);
#if defined (__cplusplus)
}
#endif

512
lib/zstdhc.c Normal file
View File

@ -0,0 +1,512 @@
/*
ZSTD HC - High Compression Mode of Zstandard
Copyright (C) 2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Zstd source repository : https://www.zstd.net
*/
/* *************************************
* Includes
***************************************/
#include <stdlib.h> /* malloc */
#include <string.h> /* memset */
#include "zstdhc.h"
#include "zstd_static.h"
#include "mem.h"
/* *************************************
* Tuning Parameter
***************************************/
static const U32 ZSTD_HC_compressionLevel_default = 9;
/* *************************************
* Local Constants
***************************************/
#define MINMATCH 4
#define MAX_DISTANCE 65535 /* <=== To be changed (dynamic ?) */
#define DICTIONARY_LOGSIZE 16
#define MAXD (1<<DICTIONARY_LOGSIZE)
#define MAXD_MASK (MAXD - 1)
#define HASH_LOG (DICTIONARY_LOGSIZE-1)
#define HASHTABLESIZE (1 << HASH_LOG)
#define HASH_MASK (HASHTABLESIZE - 1)
static const U32 g_maxCompressionLevel = 19;
#define KB *1024
#define MB *1024*1024
#define GB *(1ULL << 30)
/* *************************************
* Local Types
***************************************/
#define BLOCKSIZE (128 KB) /* define, for static allocation */
#define WORKPLACESIZE (BLOCKSIZE*3)
struct ZSTD_HC_CCtx_s
{
U32 hashTable[HASHTABLESIZE];
U16 chainTable[MAXD];
const BYTE* end; /* next block here to continue on current prefix */
const BYTE* base; /* All index relative to this position */
const BYTE* dictBase; /* alternate base for extDict */
BYTE* inputBuffer; /* deprecated */
U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more dict */
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 compressionLevel;
seqStore_t seqStore;
BYTE buffer[WORKPLACESIZE];
};
ZSTD_HC_CCtx* ZSTD_HC_createCCtx(void)
{
return (ZSTD_HC_CCtx*) malloc(sizeof(ZSTD_HC_CCtx));
}
size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx) { free(cctx); return 0; }
static void ZSTD_HC_resetCCtx (ZSTD_HC_CCtx* zc, U32 compressionLevel, const BYTE* start)
{
if (compressionLevel==0) compressionLevel = ZSTD_HC_compressionLevel_default;
if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel;
memset(zc->hashTable, 0, sizeof(zc->hashTable));
memset(zc->chainTable, 0xFF, sizeof(zc->chainTable));
zc->nextToUpdate = 64 KB;
zc->base = start - 64 KB;
zc->end = start;
zc->dictBase = start - 64 KB;
zc->dictLimit = 64 KB;
zc->lowLimit = 64 KB;
zc->compressionLevel = compressionLevel;
zc->seqStore.buffer = zc->buffer;
zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
zc->seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + (BLOCKSIZE>>2));
zc->seqStore.litStart = zc->seqStore.offCodeStart + (BLOCKSIZE>>2);
zc->seqStore.litLengthStart = zc->seqStore.litStart + BLOCKSIZE;
zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (BLOCKSIZE>>2);
zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (BLOCKSIZE>>2);
}
/* *************************************
* Local Macros
***************************************/
#define HASH_FUNCTION(u) (((u) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
//#define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] /* flexible, MAXD dependent */
#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */
static U32 ZSTD_HC_hashPtr(const void* ptr) { return HASH_FUNCTION(MEM_read32(ptr)); }
/* *************************************
* HC Compression
***************************************/
/* Update chains up to ip (excluded) */
static void ZSTD_HC_insert (ZSTD_HC_CCtx* zc, const BYTE* ip)
{
U16* chainTable = zc->chainTable;
U32* HashTable = zc->hashTable;
const BYTE* const base = zc->base;
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
while(idx < target)
{
U32 h = ZSTD_HC_hashPtr(base+idx);
size_t delta = idx - HashTable[h];
if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
DELTANEXTU16(idx) = (U16)delta;
HashTable[h] = idx;
idx++;
}
zc->nextToUpdate = target;
}
static size_t ZSTD_HC_insertAndFindBestMatch (
ZSTD_HC_CCtx* zc, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos,
const U32 maxNbAttempts)
{
U16* const chainTable = zc->chainTable;
U32* const HashTable = zc->hashTable;
const BYTE* const base = zc->base;
const BYTE* const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit;
const U32 lowLimit = (zc->lowLimit + 64 KB > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (64 KB - 1);
U32 matchIndex;
const BYTE* match;
int nbAttempts=maxNbAttempts;
size_t ml=0;
/* HC4 match finder */
ZSTD_HC_insert(zc, ip);
matchIndex = HashTable[ZSTD_HC_hashPtr(ip)];
while ((matchIndex>=lowLimit) && (nbAttempts))
{
nbAttempts--;
if (matchIndex >= dictLimit)
{
match = base + matchIndex;
if (*(match+ml) == *(ip+ml)
&& (MEM_read32(match) == MEM_read32(ip)))
{
const size_t mlt = ZSTD_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = match; }
}
}
else
{
match = dictBase + matchIndex;
if (MEM_read32(match) == MEM_read32(ip))
{
size_t mlt;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iLimit) vLimit = iLimit;
mlt = ZSTD_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
if ((ip+mlt == vLimit) && (vLimit < iLimit))
mlt += ZSTD_count(ip+mlt, base+dictLimit, iLimit);
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
}
}
matchIndex -= DELTANEXTU16(matchIndex);
}
return ml;
}
size_t ZSTD_HC_InsertAndGetWiderMatch (
ZSTD_HC_CCtx* zc,
const BYTE* const ip,
const BYTE* const iLowLimit,
const BYTE* const iHighLimit,
size_t longest,
const BYTE** matchpos,
const BYTE** startpos,
const int maxNbAttempts)
{
U16* const chainTable = zc->chainTable;
U32* const HashTable = zc->hashTable;
const BYTE* const base = zc->base;
const U32 dictLimit = zc->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const U32 lowLimit = (zc->lowLimit + 64 KB > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (64 KB - 1);
const BYTE* const dictBase = zc->dictBase;
U32 matchIndex;
int nbAttempts = maxNbAttempts;
int delta = (int)(ip-iLowLimit);
/* First Match */
ZSTD_HC_insert(zc, ip);
matchIndex = HashTable[ZSTD_HC_hashPtr(ip)];
while ((matchIndex>=lowLimit) && (nbAttempts))
{
nbAttempts--;
if (matchIndex >= dictLimit)
{
const BYTE* matchPtr = base + matchIndex;
if (*(iLowLimit + longest) == *(matchPtr - delta + longest))
if (MEM_read32(matchPtr) == MEM_read32(ip))
{
size_t mlt = MINMATCH + ZSTD_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
int back = 0;
while ((ip+back>iLowLimit)
&& (matchPtr+back > lowPrefixPtr)
&& (ip[back-1] == matchPtr[back-1]))
back--;
mlt -= back;
if (mlt > longest)
{
longest = mlt;
*matchpos = matchPtr+back;
*startpos = ip+back;
}
}
}
else
{
const BYTE* matchPtr = dictBase + matchIndex;
if (MEM_read32(matchPtr) == MEM_read32(ip))
{
size_t mlt;
int back=0;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
mlt = ZSTD_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
mlt += ZSTD_count(ip+mlt, base+dictLimit, iHighLimit);
while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--;
mlt -= back;
if (mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
}
}
matchIndex -= DELTANEXTU16(matchIndex);
}
return longest;
}
static size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart + 1;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
size_t offset_2=4, offset_1=4;
const U32 maxSearches = 1 << ctx->compressionLevel;
/* init */
ZSTD_resetSeqStore(seqStorePtr);
/* Main Search Loop */
while (ip < ilimit)
{
const BYTE* match;
size_t matchLength = ZSTD_HC_insertAndFindBestMatch(ctx, ip, ilimit, &match, maxSearches);
if (!matchLength) { ip++; continue; }
/* save match */
{
size_t litLength = ip-anchor;
size_t offset = ip-match;
if (litLength) offset_2 = offset_1;
if (offset == offset_2) offset = 0;
offset_2 = offset_1;
offset_1 = ip-match;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH);
ip += matchLength;
anchor = ip;
}
}
/* Last Literals */
{
size_t lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize;
}
/* Finale compression stage */
return ZSTD_compressSequences((BYTE*)dst, maxDstSize,
seqStorePtr, srcSize);
}
static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
static const size_t blockSize = 128 KB;
size_t remaining = srcSize;
const BYTE* ip = (const BYTE*)src;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
BYTE* const oend = op + maxDstSize;
while (remaining > blockSize)
{
size_t cSize = ZSTD_HC_compressBlock(ctxPtr, op+3, oend-op, ip, blockSize);
if (cSize == 0)
{
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
}
else
{
op[0] = (BYTE)(cSize>>16);
op[1] = (BYTE)(cSize>>8);
op[2] = (BYTE)cSize;
op[0] += (BYTE)(bt_compressed << 6); /* is a compressed block */
cSize += 3;
}
remaining -= blockSize;
ip += blockSize;
op += cSize;
if (ZSTD_isError(cSize)) return cSize;
}
/* last block */
{
size_t cSize = ZSTD_HC_compressBlock(ctxPtr, op+3, oend-op, ip, remaining);
if (cSize == 0)
{
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, remaining); /* block is not compressible */
}
else
{
op[0] = (BYTE)(cSize>>16);
op[1] = (BYTE)(cSize>>8);
op[2] = (BYTE)cSize;
op[0] += (BYTE)(bt_compressed << 6); /* is a compressed block */
cSize += 3;
}
op += cSize;
if (ZSTD_isError(cSize)) return cSize;
}
return op-ostart;
}
size_t ZSTD_HC_loadDict(ZSTD_HC_CCtx* ctx, const void* dictionary, size_t dictSize)
{
/* TBD */
(void)ctx; (void)dictionary; (void)dictSize;
return 0;
}
static void ZSTD_HC_setExternalDict(ZSTD_HC_CCtx* ctxPtr, const void* newBlock)
{
if (ctxPtr->end >= ctxPtr->base + 4)
ZSTD_HC_insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
/* Only one memory segment for extDict, so any previous extDict is lost at this stage */
ctxPtr->lowLimit = ctxPtr->dictLimit;
ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
ctxPtr->dictBase = ctxPtr->base;
ctxPtr->base = newBlock - ctxPtr->dictLimit;
ctxPtr->end = newBlock;
ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
}
size_t ZSTD_HC_compress_continue (ZSTD_HC_CCtx* ctxPtr,
void* dst, size_t dstSize,
const void* src, size_t srcSize)
{
/* Check overflow */
if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB)
{
size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
if (dictSize > 64 KB) dictSize = 64 KB;
ZSTD_HC_loadDict(ctxPtr, ctxPtr->end - dictSize, dictSize);
}
/* Check if blocks follow each other */
if ((const BYTE*)src != ctxPtr->end)
ZSTD_HC_setExternalDict(ctxPtr, (const BYTE*)src);
/* Check overlapping src/dictionary space (typical of cycling buffers) */
{
const BYTE* sourceEnd = (const BYTE*) src + srcSize;
const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd))
{
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
}
}
return ZSTD_HC_compress_generic (ctxPtr, dst, dstSize, src, srcSize);
}
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, unsigned compressionLevel, const void* src)
{
/* Sanity check */
if (maxDstSize < 4) return ERROR(dstSize_tooSmall);
/* Init */
ZSTD_HC_resetCCtx(ctx, compressionLevel, src);
/* Write Header */
MEM_writeLE32(dst, ZSTD_magicNumber);
return 4;
}
size_t ZSTD_HC_compressCCtx (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel)
{
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
/* Header */
size_t oSize = ZSTD_HC_compressBegin(ctx, dst, maxDstSize, compressionLevel, src);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
maxDstSize -= oSize;
/* body (compression) */
op += ZSTD_HC_compress_generic (ctx, op, maxDstSize, src, srcSize);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
maxDstSize -= oSize;
/* Close frame */
oSize = ZSTD_compressEnd((ZSTD_CCtx*)ctx, op, maxDstSize);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
return (op - ostart);
}
size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel)
{
ZSTD_HC_CCtx ctxBody;
return ZSTD_HC_compressCCtx(&ctxBody, dst, maxDstSize, src, srcSize, compressionLevel);
}
/**************************************
* Streaming Functions
**************************************/
/* dictionary saving */
size_t ZSTD_HC_saveDict (ZSTD_HC_CCtx* ctx, void* safeBuffer, size_t dictSize)
{
/* TBD */
(void)ctx; (void)safeBuffer; (void)dictSize;
return 0;
}

76
lib/zstdhc.h Normal file
View File

@ -0,0 +1,76 @@
/*
zstdhc - high compression variant
Header File
Copyright (C) 2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : http://www.zstd.net
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/* *************************************
* Includes
***************************************/
#include <stddef.h> /* size_t */
/* *************************************
* Simple function
***************************************/
/**
ZSTD_HC_compress() :
Compresses 'srcSize' bytes from buffer 'src' into buffer 'dst', of maximum size 'dstSize'.
Destination buffer must be already allocated.
Compression runs faster if maxDstSize >= ZSTD_compressBound(srcSize).
@return : the number of bytes written into buffer 'dst'
or an error code if it fails (which can be tested using ZSTD_isError())
*/
size_t ZSTD_HC_compress(void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
unsigned compressionLevel);
/* *************************************
* Advanced functions
***************************************/
typedef struct ZSTD_HC_CCtx_s ZSTD_HC_CCtx; /* incomplete type */
ZSTD_HC_CCtx* ZSTD_HC_createCCtx(void);
size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx);
/**
ZSTD_HC_compressCCtx() :
Same as ZSTD_compress(), but requires a ZSTD_HC_CCtx working space already allocated
*/
size_t ZSTD_HC_compressCCtx(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel);
#if defined (__cplusplus)
}
#endif

4
programs/Makefile
View File

@ -30,7 +30,7 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
VERSION?= v0.2.0
VERSION?= 0.3.0
DESTDIR?=
PREFIX ?= /usr/local
@ -58,7 +58,7 @@ default: zstd
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 datagen
zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c

90
programs/bench.c
View File

@ -23,29 +23,29 @@
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/***************************************
/* **************************************
* Compiler Options
***************************************/
****************************************/
/* Disable some Visual warning messages */
#define _CRT_SECURE_NO_WARNINGS /* fopen */
// Unix Large Files support (>4GB)
/* Unix Large Files support (>4GB) */
#define _FILE_OFFSET_BITS 64
#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */
# define _LARGEFILE_SOURCE
#elif ! defined(__LP64__) // No point defining Large file for 64 bit
#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */
# define _LARGEFILE64_SOURCE
#endif
// S_ISREG & gettimeofday() are not supported by MSVC
/* S_ISREG & gettimeofday() are not supported by MSVC */
#if defined(_MSC_VER) || defined(_WIN32)
# define BMK_LEGACY_TIMER 1
#endif
/**************************************
/* *************************************
* Includes
**************************************/
***************************************/
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> // fprintf, fopen, ftello64
@ -59,40 +59,23 @@
# include <sys/time.h> // gettimeofday
#endif
#include "mem.h"
#include "zstd.h"
#include "zstdhc.h"
#include "xxhash.h"
/**************************************
/* *************************************
* Compiler specifics
**************************************/
***************************************/
#if !defined(S_ISREG)
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
/**************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/**************************************
/* *************************************
* Constants
**************************************/
***************************************/
#define NBLOOPS 3
#define TIMELOOP 2500
@ -108,15 +91,15 @@ static U32 prime1 = 2654435761U;
static U32 prime2 = 2246822519U;
/**************************************
/* *************************************
* Macros
**************************************/
***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
/**************************************
/* *************************************
* Benchmark Parameters
**************************************/
***************************************/
static int nbIterations = NBLOOPS;
static size_t g_blockSize = 0;
@ -133,9 +116,9 @@ void BMK_SetBlockSize(size_t blockSize)
}
/*********************************************************
/* ********************************************************
* Private functions
*********************************************************/
**********************************************************/
#if defined(BMK_LEGACY_TIMER)
@ -176,10 +159,9 @@ static int BMK_GetMilliSpan( int nTimeStart )
}
/*********************************************************
/* ********************************************************
* Data generator
*********************************************************/
**********************************************************/
/* will hopefully be converted into ROL instruction by compiler */
static U32 BMK_rotl32(unsigned val32, unsigned nbBits) { return((val32 << nbBits) | (val32 >> (32 - nbBits))); }
@ -234,9 +216,9 @@ static void BMK_datagen(void* buffer, size_t bufferSize, double proba, U32 seed)
}
/*********************************************************
/* ********************************************************
* Bench functions
*********************************************************/
**********************************************************/
typedef struct
{
char* srcPtr;
@ -249,6 +231,14 @@ typedef struct
} blockParam_t;
typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel);
static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel)
{
(void)compressionLevel;
return ZSTD_compress(dst, maxDstSize, src, srcSize);
}
#define MIN(a,b) (a<b ? a : b)
static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
@ -259,10 +249,12 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
compressor_t compressor;
U64 crcOrig;
/* Init */
(void)cLevel;
if (cLevel <= 1) compressor = local_compress_fast;
else compressor = ZSTD_HC_compress;
/* Memory allocation & restrictions */
if (!compressedBuffer || !resultBuffer || !blockTable)
@ -328,8 +320,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
while (BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
for (blockNb=0; blockNb<nbBlocks; blockNb++)
blockTable[blockNb].cSize = ZSTD_compress(blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
blockTable[blockNb].cSize = compressor(blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel);
nbLoops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
@ -364,13 +355,14 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck)
{
unsigned i;
unsigned u;
unsigned eBlockSize = MIN(65536*2, blockSize);
DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck);
for (i=0; i<srcSize; i++)
for (u=0; u<srcSize; u++)
{
if (((BYTE*)srcBuffer)[i] != ((BYTE*)resultBuffer)[i])
if (((BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u])
{
printf("\nDecoding error at pos %u \n", i);
printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
break;
}
}

9
programs/fileio.c
View File

@ -116,12 +116,6 @@ static const unsigned FIO_blockHeaderSize = 3;
#define CACHELINE 64
/**************************************
* Complex types
**************************************/
typedef enum { bt_compressed, bt_raw, bt_rle, bt_crc } bType_t;
/**************************************
* Macros
**************************************/
@ -218,7 +212,7 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu
}
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename)
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, unsigned cLevel)
{
U64 filesize = 0;
U64 compressedfilesize = 0;
@ -236,6 +230,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
/* Init */
(void)cLevel;
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
/* Allocate Memory */

20
programs/fileio.h
View File

@ -29,7 +29,7 @@ extern "C" {
#endif
/**************************************
/* *************************************
* Special i/o constants
**************************************/
#define nullString "null"
@ -42,28 +42,28 @@ extern "C" {
#endif
/**************************************
/* *************************************
* Parameters
**************************************/
***************************************/
void FIO_overwriteMode(void);
void FIO_setNotificationLevel(unsigned level);
/**************************************
/* *************************************
* Stream/File functions
**************************************/
unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename);
***************************************/
unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, unsigned cLevel);
unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename);
/*
/**
FIO_compressFilename :
result : size of compressed file
@result : size of compressed file
FIO_decompressFilename :
result : size of regenerated file
@result : size of regenerated file
*/
#if defined (__cplusplus)
}
#endif
#endif

18
programs/zstdcli.c
View File

@ -169,6 +169,7 @@ int main(int argc, char** argv)
main_pause=0;
unsigned fileNameStart = 0;
unsigned nbFiles = 0;
unsigned cLevel = 0;
const char* programName = argv[0];
const char* inFileName = NULL;
const char* outFileName = NULL;
@ -217,6 +218,19 @@ int main(int argc, char** argv)
while (argument[0]!=0)
{
/* compression Level */
if ((*argument>='0') && (*argument<='9'))
{
cLevel = 0;
while ((*argument >= '0') && (*argument <= '9'))
{
cLevel *= 10;
cLevel += *argument - '0';
argument++;
}
continue;
}
switch(argument[0])
{
/* Display help */
@ -307,7 +321,7 @@ int main(int argc, char** argv)
if (!strcmp(inFileName, stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
/* Check if benchmark is selected */
if (bench) { BMK_benchFiles(argv+fileNameStart, nbFiles, 0); goto _end; }
if (bench) { BMK_benchFiles(argv+fileNameStart, nbFiles, cLevel); goto _end; }
/* No output filename ==> try to select one automatically (when possible) */
while (!outFileName)
@ -351,7 +365,7 @@ int main(int argc, char** argv)
if (decode)
FIO_decompressFilename(outFileName, inFileName);
else
FIO_compressFilename(outFileName, inFileName);
FIO_compressFilename(outFileName, inFileName, cLevel);
_end:
if (main_pause) waitEnter();