krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-31 16:47:48 +02:00
Code Issues Releases 1 Activity

updated fse

Browse Source
This commit is contained in:
Yann Collet
2016-01-06 01:58:37 +01:00
parent 8ab94b653c
commit 3b994cb465
6 changed files with 198 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
lib/fse.c
View File

@@ -34,10 +34,10 @@
#ifndef FSE_COMMONDEFS_ONLY
/****************************************************************
/* **************************************************************
* Tuning parameters
****************************************************************/
/* MEMORY_USAGE :
/*!MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
@@ -45,26 +45,23 @@
#define FSE_MAX_MEMORY_USAGE 14
#define FSE_DEFAULT_MEMORY_USAGE 13
/* FSE_MAX_SYMBOL_VALUE :
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized.
* Required for proper stack allocation */
#define FSE_MAX_SYMBOL_VALUE 255
/****************************************************************
/* **************************************************************
* template functions type & suffix
****************************************************************/
#define FSE_FUNCTION_TYPE BYTE
#define FSE_FUNCTION_EXTENSION
#define FSE_DECODE_TYPE FSE_decode_t
/****************************************************************
* Byte symbol type
****************************************************************/
#endif /* !FSE_COMMONDEFS_ONLY */
/****************************************************************
/* **************************************************************
* Compiler specifics
****************************************************************/
#ifdef _MSC_VER /* Visual Studio */
@@ -82,7 +79,7 @@
#endif
/****************************************************************
/* **************************************************************
* Includes
****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
@@ -92,7 +89,7 @@
#include "fse_static.h"
/****************************************************************
/* ***************************************************************
* Constants
*****************************************************************/
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
@@ -107,20 +104,20 @@
#endif
/****************************************************************
/* **************************************************************
* Error Management
****************************************************************/
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/****************************************************************
/* **************************************************************
* Complex types
****************************************************************/
typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
/****************************************************************
/* **************************************************************
* Templates
****************************************************************/
/*
@@ -144,8 +141,7 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
/* Function templates */
size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION)
(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned safe)
size_t FSE_count_generic(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned safe)
{
const FSE_FUNCTION_TYPE* ip = source;
const FSE_FUNCTION_TYPE* const iend = ip+sourceSize;
@@ -226,7 +222,7 @@ size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION)
size_t FSE_FUNCTION_NAME(FSE_countFast, FSE_FUNCTION_EXTENSION)
(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize)
{
return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0);
return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 0);
}
size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION)
@@ -235,25 +231,26 @@ size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION)
if ((sizeof(FSE_FUNCTION_TYPE)==1) && (*maxSymbolValuePtr >= 255))
{
*maxSymbolValuePtr = 255;
return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0);
return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 0);
}
return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 1);
return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 1);
}
static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; }
size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
const unsigned tableSize = 1 << tableLog;
const unsigned tableMask = tableSize - 1;
U16* tableU16 = ( (U16*) ct) + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) (((U32*)ct) + 1 + (tableLog ? tableSize>>1 : 1) );
void* const ptr = ct;
U16* const tableU16 = ( (U16*) ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
const unsigned step = FSE_tableStep(tableSize);
unsigned cumul[FSE_MAX_SYMBOL_VALUE+2];
U32 position = 0;
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* init isn't necessary, even if static analyzer complain about it */
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
U32 highThreshold = tableSize-1;
unsigned symbol;
unsigned i;
@@ -269,7 +266,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
cumul[0] = 0;
for (i=1; i<=maxSymbolValue+1; i++)
{
if (normalizedCounter[i-1]==-1) /* Low prob symbol */
if (normalizedCounter[i-1]==-1) /* Low proba symbol */
{
cumul[i] = cumul[i-1] + 1;
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(i-1);
@@ -287,7 +284,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
{
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
position = (position + step) & tableMask;
while (position > highThreshold) position = (position + step) & tableMask; /* Lowprob area */
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */
}
}
@@ -296,7 +293,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
/* Build table */
for (i=0; i<tableSize; i++)
{
FSE_FUNCTION_TYPE s = tableSymbol[i]; /* static analyzer doesn't understand tableSymbol is properly initialized */
FSE_FUNCTION_TYPE s = tableSymbol[i]; /* note : static analyzer may not understand tableSymbol is properly initialized */
tableU16[cumul[s]++] = (U16) (tableSize+i); /* TableU16 : sorted by symbol order; gives next state value */
}
@@ -332,24 +329,22 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
}
#define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION)
FSE_DTable* FSE_FUNCTION_NAME(FSE_createDTable, FSE_FUNCTION_EXTENSION) (unsigned tableLog)
FSE_DTable* FSE_createDTable (unsigned tableLog)
{
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
}
void FSE_FUNCTION_NAME(FSE_freeDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt)
void FSE_freeDTable (FSE_DTable* dt)
{
free(dt);
}
size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */
FSE_DTableHeader DTableH;
void* const tdPtr = dt+1; /* because dt is unsigned, 32-bits aligned on 32-bits */
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
const U32 tableSize = 1 << tableLog;
const U32 tableMask = tableSize-1;
const U32 step = FSE_tableStep(tableSize);
@@ -365,7 +360,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
/* Init, lay down lowprob symbols */
DTableH[0].tableLog = (U16)tableLog;
DTableH.tableLog = (U16)tableLog;
for (s=0; s<=maxSymbolValue; s++)
{
if (normalizedCounter[s]==-1)
@@ -406,7 +401,8 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
}
}
DTableH->fastMode = (U16)noLarge;
DTableH.fastMode = (U16)noLarge;
memcpy(dt, &DTableH, sizeof(DTableH));
return 0;
}
@@ -890,8 +886,10 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask;
U16* tableU16 = ( (U16*) ct) + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((((U32*)ct)+1) + (tableSize>>1));
void* const ptr = ct;
U16* const tableU16 = ( (U16*) ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
unsigned s;
/* Sanity checks */
@@ -918,8 +916,10 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
/* fake FSE_CTable, for rle (100% always same symbol) input */
size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
{
U16* tableU16 = ( (U16*) ct) + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((U32*)ct + 2);
void* ptr = ct;
U16* tableU16 = ( (U16*) ptr) + 2;
void* FSCTptr = (U32*)ptr + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr;
/* header */
tableU16[-2] = (U16) 0;
@@ -1076,8 +1076,10 @@ size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize)
*********************************************************/
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
void* dPtr = dt + 1;
FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
DTableH->tableLog = 0;
DTableH->fastMode = 0;
@@ -1092,8 +1094,10 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
void* dPtr = dt + 1;
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask;
@@ -1189,7 +1193,8 @@ size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt)
{
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt;
const void* ptr = dt;
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
const U32 fastMode = DTableH->fastMode;
/* select fast mode (static) */

46
lib/fse.h
View File

@@ -40,20 +40,20 @@ extern "C" {
#endif
/******************************************
/* *****************************************
* Includes
******************************************/
#include <stddef.h> /* size_t, ptrdiff_t */
/******************************************
/* *****************************************
* FSE simple functions
******************************************/
size_t FSE_compress(void* dst, size_t maxDstSize,
const void* src, size_t srcSize);
size_t FSE_decompress(void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize);
/*
/*!
FSE_compress():
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= FSE_compressBound(srcSize)
@@ -74,7 +74,7 @@ FSE_decompress():
*/
/******************************************
/* *****************************************
* Tool functions
******************************************/
size_t FSE_compressBound(size_t size); /* maximum compressed size */
@@ -84,10 +84,10 @@ unsigned FSE_isError(size_t code); /* tells if a return value is an er
const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
/******************************************
/* *****************************************
* FSE advanced functions
******************************************/
/*
/*!
FSE_compress2():
Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
Both parameters can be defined as '0' to mean : use default value
@@ -99,10 +99,10 @@ FSE_compress2():
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
/******************************************
/* *****************************************
* FSE detailed API
******************************************/
/*
/*!
FSE_compress() does the following:
1. count symbol occurrence from source[] into table count[]
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
@@ -122,7 +122,7 @@ or to save and provide normalized distribution using external method.
/* *** COMPRESSION *** */
/*
/*!
FSE_count():
Provides the precise count of each symbol within a table 'count'
'count' is a table of unsigned int, of minimum size (maxSymbolValuePtr[0]+1).
@@ -132,14 +132,14 @@ FSE_count():
if FSE_isError(return), it's an error code. */
size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize);
/*
/*!
FSE_optimalTableLog():
dynamically downsize 'tableLog' when conditions are met.
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
return : recommended tableLog (necessarily <= initial 'tableLog') */
unsigned FSE_optimalTableLog(unsigned tableLog, size_t srcSize, unsigned maxSymbolValue);
/*
/*!
FSE_normalizeCount():
normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
@@ -147,13 +147,13 @@ FSE_normalizeCount():
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
/*
/*!
FSE_NCountWriteBound():
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'
Typically useful for allocation purpose. */
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
/*
/*!
FSE_writeNCount():
Compactly save 'normalizedCounter' into 'buffer'.
return : size of the compressed table
@@ -161,21 +161,21 @@ FSE_writeNCount():
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*
/*!
Constructor and Destructor of type FSE_CTable
Note that its size depends on 'tableLog' and 'maxSymbolValue' */
typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void* */
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
void FSE_freeCTable (FSE_CTable* ct);
/*
/*!
FSE_buildCTable():
Builds 'ct', which must be already allocated, using FSE_createCTable()
return : 0
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*
/*!
FSE_compress_usingCTable():
Compress 'src' using 'ct' into 'dst' which must be already allocated
return : size of compressed data (<= maxDstSize)
@@ -183,7 +183,7 @@ FSE_compress_usingCTable():
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_compress_usingCTable (void* dst, size_t maxDstSize, const void* src, size_t srcSize, const FSE_CTable* ct);
/*
/*!
Tutorial :
----------
The first step is to count all symbols. FSE_count() does this job very fast.
@@ -229,7 +229,7 @@ If there is an error, the function will return an ErrorCode (which can be tested
/* *** DECOMPRESSION *** */
/*
/*!
FSE_readNCount():
Read compactly saved 'normalizedCounter' from 'rBuffer'.
return : size read from 'rBuffer'
@@ -237,21 +237,21 @@ FSE_readNCount():
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
/*
/*!
Constructor and Destructor of type FSE_DTable
Note that its size depends on 'tableLog' */
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
FSE_DTable* FSE_createDTable(unsigned tableLog);
void FSE_freeDTable(FSE_DTable* dt);
/*
/*!
FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable()
return : 0,
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*
/*!
FSE_decompress_usingDTable():
Decompress compressed source 'cSrc' of size 'cSrcSize' using 'dt'
into 'dst' which must be already allocated.
@@ -259,7 +259,7 @@ FSE_decompress_usingDTable():
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
/*
/*!
Tutorial :
----------
(Note : these functions only decompress FSE-compressed blocks.

53
lib/fse_static.h
View File

@@ -40,31 +40,31 @@ extern "C" {
#endif
/******************************************
* FSE API compatible with DLL
******************************************/
/* *****************************************
* Dependencies
*******************************************/
#include "fse.h"
#include "bitstream.h"
/******************************************
/* *****************************************
* Static allocation
******************************************/
*******************************************/
/* FSE buffer bounds */
#define FSE_NCOUNTBOUND 512
#define FSE_BLOCKBOUND(size) (size + (size>>7))
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* You can statically allocate FSE CTable/DTable as a table of unsigned using below macro */
/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
/******************************************
/* *****************************************
* FSE advanced API
******************************************/
*******************************************/
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize);
/* same as FSE_count(), but blindly trust that all values within src are <= maxSymbolValuePtr[0] */
/* same as FSE_count(), but blindly trust that all values within src are <= *maxSymbolValuePtr */
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
/* build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
@@ -79,10 +79,10 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
/* build a fake FSE_DTable, designed to always generate the same symbolValue */
/******************************************
/* *****************************************
* FSE symbol compression API
******************************************/
/*
*******************************************/
/*!
This API consists of small unitary functions, which highly benefit from being inlined.
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
Visual seems to do it automatically.
@@ -103,7 +103,7 @@ static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsig
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
/*
/*!
These functions are inner components of FSE_compress_usingCTable().
They allow the creation of custom streams, mixing multiple tables and bit sources.
@@ -147,9 +147,9 @@ If there is an error, it returns an errorCode (which can be tested using FSE_isE
*/
/******************************************
/* *****************************************
* FSE symbol decompression API
******************************************/
*******************************************/
typedef struct
{
size_t state;
@@ -163,7 +163,7 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
/*
/*!
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
You will decode FSE-encoded symbols from the bitStream,
and also any other bitFields you put in, **in reverse order**.
@@ -213,16 +213,16 @@ Check also the states. There might be some symbols left there, if some high prob
*/
/******************************************
/* *****************************************
* FSE unsafe API
******************************************/
*******************************************/
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
/******************************************
* Implementation of inline functions
******************************************/
/* *****************************************
* Implementation of inlined functions
*******************************************/
typedef struct
{
int deltaFindState;
@@ -231,10 +231,12 @@ typedef struct
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
{
const U32 tableLog = ( (const U16*) ct) [0];
const void* ptr = ct;
const U16* u16ptr = (const U16*) ptr;
const U32 tableLog = *u16ptr;
statePtr->value = (ptrdiff_t)1<<tableLog;
statePtr->stateTable = ((const U16*) ct) + 2;
statePtr->symbolTT = (const void*)((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
statePtr->stateTable = u16ptr+2;
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
statePtr->stateLog = tableLog;
}
@@ -269,7 +271,8 @@ typedef struct
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
{
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt;
const void* ptr = dt;
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;

81
lib/huff0.c
View File

@@ -32,7 +32,7 @@
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
/****************************************************************
/* **************************************************************
* Compiler specifics
****************************************************************/
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@@ -57,7 +57,7 @@
#endif
/****************************************************************
/* **************************************************************
* Includes
****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
@@ -68,7 +68,7 @@
#include "fse.h" /* header compression */
/****************************************************************
/* **************************************************************
* Constants
****************************************************************/
#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
@@ -80,27 +80,21 @@
#endif
/****************************************************************
/* **************************************************************
* Error Management
****************************************************************/
unsigned HUF_isError(size_t code) { return ERR_isError(code); }
const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/******************************************
* Helper functions
******************************************/
unsigned HUF_isError(size_t code) { return ERR_isError(code); }
const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
/*********************************************************
/* *******************************************************
* Huff0 : Huffman block compression
*********************************************************/
typedef struct HUF_CElt_s {
struct HUF_CElt_s {
U16 val;
BYTE nbBits;
} HUF_CElt ;
}; /* typedef'd to HUF_CElt within huff0_static.h */
typedef struct nodeElt_s {
U32 count;
@@ -320,7 +314,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
/* sort, decreasing order */
HUF_sort(huffNode, count, maxSymbolValue);
// init for parents
/* init for parents */
nonNullRank = maxSymbolValue;
while(huffNode[nonNullRank].count == 0) nonNullRank--;
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
@@ -330,7 +324,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
huffNode0[0].count = (U32)(1U<<31);
// create parents
/* create parents */
while (nodeNb <= nodeRoot)
{
U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
@@ -340,7 +334,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
nodeNb++;
}
// distribute weights (unlimited tree height)
/* distribute weights (unlimited tree height) */
huffNode[nodeRoot].nbBits = 0;
for (n=nodeRoot-1; n>=STARTNODE; n--)
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
@@ -368,9 +362,9 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
}
}
for (n=0; n<=maxSymbolValue; n++)
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; // push nbBits per symbol, symbol order
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
for (n=0; n<=maxSymbolValue; n++)
tree[n].val = valPerRank[tree[n].nbBits]++; // assign value within rank, symbol order
tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
}
return maxNbBits;
@@ -636,12 +630,12 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0;
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
size_t iSize;
U32 nbSymbols = 0;
U32 n;
U32 nextRankStart;
HUF_DEltX2* const dt = (HUF_DEltX2*)(DTable + 1);
void* const dtPtr = DTable + 1;
HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
@@ -730,7 +724,8 @@ size_t HUF_decompress1X2_usingDTable(
BYTE* const oend = op + dstSize;
size_t errorCode;
const U32 dtLog = DTable[0];
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1;
const void* dtPtr = DTable;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr)+1;
BIT_DStream_t bitD;
errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode)) return errorCode;
@@ -770,8 +765,8 @@ size_t HUF_decompress4X2_usingDTable(
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1;
const void* const dtPtr = DTable;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
@@ -978,9 +973,9 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
rankVal_t rankVal;
U32 tableLog, maxW, sizeOfSort, nbSymbols;
const U32 memLog = DTable[0];
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
HUF_DEltX4* const dt = ((HUF_DEltX4*)DTable) + 1;
size_t iSize;
void* dtPtr = DTable;
HUF_DEltX4* const dt = ((HUF_DEltX4*)dtPtr) + 1;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
@@ -1127,7 +1122,8 @@ size_t HUF_decompress1X4_usingDTable(
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1;
const void* const dtPtr = DTable;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1;
size_t errorCode;
/* Init */
@@ -1170,8 +1166,8 @@ size_t HUF_decompress4X4_usingDTable(
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1;
const void* const dtPtr = DTable;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
@@ -1352,8 +1348,7 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
U32 tableLog, maxW, sizeOfSort, nbSymbols;
rankVal_t rankVal;
const U32 memLog = DTable[0];
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
size_t iSize;
if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
@@ -1418,8 +1413,10 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
/* fill tables */
{
HUF_DDescX6* DDescription = (HUF_DDescX6*)(DTable+1);
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(memLog-1)));
void* ddPtr = DTable+1;
HUF_DDescX6* DDescription = (HUF_DDescX6*)ddPtr;
void* dsPtr = DTable + 1 + ((size_t)1<<(memLog-1));
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)dsPtr;
HUF_DSeqX6 DSeq;
HUF_DDescX6 DDesc;
DSeq.sequence = 0;
@@ -1478,8 +1475,10 @@ static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStr
static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog)
{
const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1);
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1)));
const void* const ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
BYTE* const pStart = p;
/* up to 16 symbols at a time */
@@ -1557,8 +1556,10 @@ size_t HUF_decompress4X6_usingDTable(
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1);
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1)));
const void* const ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
size_t errorCode;
/* Init */

16
lib/huff0.h
View File

@@ -40,25 +40,25 @@ extern "C" {
#endif
/******************************************
/* ****************************************
* Dependency
******************************************/
#include <stddef.h> /* size_t */
/******************************************
/* ****************************************
* Huff0 simple functions
******************************************/
size_t HUF_compress(void* dst, size_t maxDstSize,
const void* src, size_t srcSize);
size_t HUF_decompress(void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize);
/*
/*!
HUF_compress():
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster if maxDstSize >= HUF_compressBound(srcSize).
Note : srcSize must be <= 128 KB
return : size of compressed data (<= maxDstSize)
@return : size of compressed data (<= maxDstSize)
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single repeated byte symbol (RLE compression)
if HUF_isError(return), compression failed (more details using HUF_getErrorName())
@@ -68,12 +68,12 @@ HUF_decompress():
into already allocated destination buffer 'dst', of size 'dstSize'.
'dstSize' must be the exact size of original (uncompressed) data.
Note : in contrast with FSE, HUF_decompress can regenerate RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, because it knows size to regenerate.
return : size of regenerated data (== dstSize)
or an error code, which can be tested using HUF_isError()
@return : size of regenerated data (== dstSize)
or an error code, which can be tested using HUF_isError()
*/
/******************************************
/* ****************************************
* Tool functions
******************************************/
size_t HUF_compressBound(size_t size); /* maximum compressed size */
@@ -83,7 +83,7 @@ unsigned HUF_isError(size_t code); /* tells if a return value is an er
const char* HUF_getErrorName(size_t code); /* provides error code string (useful for debugging) */
/******************************************
/* ****************************************
* Advanced functions
******************************************/
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);

51
lib/huff0_static.h
View File

@@ -40,13 +40,13 @@ extern "C" {
#endif
/******************************************
/* ****************************************
* Dependency
******************************************/
#include "huff0.h"
/******************************************
/* ****************************************
* Static allocation macros
******************************************/
/* Huff0 buffer bounds */
@@ -64,14 +64,57 @@ extern "C" {
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog }
/******************************************
* Advanced functions
/* ****************************************
* Advanced decompression functions
******************************************/
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder */
/* ****************************************
* Huff0 detailed API
******************************************/
/*!
HUF_compress() does the following:
1. count symbol occurrence from source[] into table count[] using FSE_count()
2. build Huffman table from count using HUF_buildCTable()
3. save Huffman table to memory buffer using HUF_writeCTable()
4. encode the data stream using HUF_compress_usingCTable()
The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
or to save and regenerate 'CTable' using external methods.
*/
/* FSE_count() : find it within "fse.h" */
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, unsigned maxSymbolValue, unsigned huffLog);
size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
/*!
HUF_decompress() does the following:
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
2. build Huffman table from save, using HUF_readDTableXn()
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
*/
size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbol decoder */
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
#if defined (__cplusplus)
}
#endif