krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-07 09:26:03 +02:00
Code Issues Releases 1 Activity

Merge pull request #1310 from facebook/Dworkspace

Browse Source 
reduce DDict size by 2KB
This commit is contained in:
Yann Collet 2018-09-10 13:11:11 -07:00 committed by GitHub
commit 51a246da82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/decompress/huf_decompress.c
View File

@ -533,9 +533,9 @@ static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
} }
} }
size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
size_t srcSize, void* workSpace, const void* src, size_t srcSize,
size_t wkspSize) void* workSpace, size_t wkspSize)
{ {
U32 tableLog, maxW, sizeOfSort, nbSymbols; U32 tableLog, maxW, sizeOfSort, nbSymbols;
DTableDesc dtd = HUF_getDTableDesc(DTable); DTableDesc dtd = HUF_getDTableDesc(DTable);

77
lib/decompress/zstd_decompress.c
View File

@ -110,11 +110,10 @@ typedef struct {
#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
typedef struct { typedef struct {
ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
U32 rep[ZSTD_REP_NUM]; U32 rep[ZSTD_REP_NUM];
} ZSTD_entropyDTables_t; } ZSTD_entropyDTables_t;
@ -125,6 +124,7 @@ struct ZSTD_DCtx_s
const ZSTD_seqSymbol* OFTptr; const ZSTD_seqSymbol* OFTptr;
const HUF_DTable* HUFptr; const HUF_DTable* HUFptr;
ZSTD_entropyDTables_t entropy; ZSTD_entropyDTables_t entropy;
U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */
const void* previousDstEnd; /* detect continuity */ const void* previousDstEnd; /* detect continuity */
const void* prefixStart; /* start of current segment */ const void* prefixStart; /* start of current segment */
const void* virtualStart; /* virtual start of previous segment if it was just before current one */ const void* virtualStart; /* virtual start of previous segment if it was just before current one */
@ -612,9 +612,9 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
HUF_decompress4X_usingDTable_bmi2(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr, dctx->bmi2) ) : HUF_decompress4X_usingDTable_bmi2(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr, dctx->bmi2) ) :
( singleStream ? ( singleStream ?
HUF_decompress1X1_DCtx_wksp_bmi2(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, HUF_decompress1X1_DCtx_wksp_bmi2(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize,
dctx->entropy.workspace, sizeof(dctx->entropy.workspace), dctx->bmi2) : dctx->workspace, sizeof(dctx->workspace), dctx->bmi2) :
HUF_decompress4X_hufOnly_wksp_bmi2(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, HUF_decompress4X_hufOnly_wksp_bmi2(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize,
dctx->entropy.workspace, sizeof(dctx->entropy.workspace), dctx->bmi2)))) dctx->workspace, sizeof(dctx->workspace), dctx->bmi2))))
return ERROR(corruption_detected); return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
@ -2196,18 +2196,26 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict
/* ZSTD_loadEntropy() : /* ZSTD_loadEntropy() :
* dict : must point at beginning of a valid zstd dictionary * dict : must point at beginning of a valid zstd dictionary
* @return : size of entropy tables read */ * @return : size of entropy tables read */
static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, const void* const dict, size_t const dictSize) static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy,
const void* const dict, size_t const dictSize)
{ {
const BYTE* dictPtr = (const BYTE*)dict; const BYTE* dictPtr = (const BYTE*)dict;
const BYTE* const dictEnd = dictPtr + dictSize; const BYTE* const dictEnd = dictPtr + dictSize;
if (dictSize <= 8) return ERROR(dictionary_corrupted); if (dictSize <= 8) return ERROR(dictionary_corrupted);
assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */
dictPtr += 8; /* skip header = magic + dictID */ dictPtr += 8; /* skip header = magic + dictID */
ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, LLTable) == 0);
{ size_t const hSize = HUF_readDTableX2_wksp( ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == sizeof(entropy->LLTable));
entropy->hufTable, dictPtr, dictEnd - dictPtr, ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == sizeof(entropy->LLTable) + sizeof(entropy->OFTable));
entropy->workspace, sizeof(entropy->workspace)); ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, hufTable) == sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, hufTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE);
{ void* const workspace = entropy; /* use fse tables as temporary workspace; implies fse table precede huffTable at beginning of entropy */
size_t const workspaceSize = offsetof(ZSTD_entropyDTables_t, hufTable);
size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
dictPtr, dictEnd - dictPtr,
workspace, workspaceSize);
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
dictPtr += hSize; dictPtr += hSize;
} }
@ -2218,7 +2226,7 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, const void* const
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
if (offcodeMaxValue > MaxOff) return ERROR(dictionary_corrupted); if (offcodeMaxValue > MaxOff) return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
ZSTD_buildFSETable(entropy->OFTable, ZSTD_buildFSETable( entropy->OFTable,
offcodeNCount, offcodeMaxValue, offcodeNCount, offcodeMaxValue,
OF_base, OF_bits, OF_base, OF_bits,
offcodeLog); offcodeLog);
@ -2231,7 +2239,7 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, const void* const
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (matchlengthMaxValue > MaxML) return ERROR(dictionary_corrupted); if (matchlengthMaxValue > MaxML) return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
ZSTD_buildFSETable(entropy->MLTable, ZSTD_buildFSETable( entropy->MLTable,
matchlengthNCount, matchlengthMaxValue, matchlengthNCount, matchlengthMaxValue,
ML_base, ML_bits, ML_base, ML_bits,
matchlengthLog); matchlengthLog);
@ -2244,7 +2252,7 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, const void* const
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (litlengthMaxValue > MaxLL) return ERROR(dictionary_corrupted); if (litlengthMaxValue > MaxLL) return ERROR(dictionary_corrupted);
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
ZSTD_buildFSETable(entropy->LLTable, ZSTD_buildFSETable( entropy->LLTable,
litlengthNCount, litlengthMaxValue, litlengthNCount, litlengthMaxValue,
LL_base, LL_bits, LL_base, LL_bits,
litlengthLog); litlengthLog);
@ -2365,7 +2373,9 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddi
return 0; return 0;
} }
static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, ZSTD_dictContentType_e dictContentType) static size_t
ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict,
ZSTD_dictContentType_e dictContentType)
{ {
ddict->dictID = 0; ddict->dictID = 0;
ddict->entropyPresent = 0; ddict->entropyPresent = 0;
@ -2386,7 +2396,9 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, ZSTD_dictContentType_e
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
/* load entropy tables */ /* load entropy tables */
CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted ); CHECK_E( ZSTD_loadEntropy(&ddict->entropy,
ddict->dictContent, ddict->dictSize),
dictionary_corrupted );
ddict->entropyPresent = 1; ddict->entropyPresent = 1;
return 0; return 0;
} }
@ -2425,14 +2437,15 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
if (!customMem.customAlloc ^ !customMem.customFree) return NULL; if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
if (!ddict) return NULL; if (ddict == NULL) return NULL;
ddict->cMem = customMem; ddict->cMem = customMem;
{ size_t const initResult = ZSTD_initDDict_internal(ddict,
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, dictLoadMethod, dictContentType) )) { dict, dictSize,
ZSTD_freeDDict(ddict); dictLoadMethod, dictContentType);
return NULL; if (ZSTD_isError(initResult)) {
} ZSTD_freeDDict(ddict);
return NULL;
} }
return ddict; return ddict;
} }
} }
@ -2459,23 +2472,25 @@ ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize
const ZSTD_DDict* ZSTD_initStaticDDict( const ZSTD_DDict* ZSTD_initStaticDDict(
void* workspace, size_t workspaceSize, void* sBuffer, size_t sBufferSize,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType) ZSTD_dictContentType_e dictContentType)
{ {
size_t const neededSpace = size_t const neededSpace = sizeof(ZSTD_DDict)
sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
ZSTD_DDict* const ddict = (ZSTD_DDict*)workspace; ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
assert(workspace != NULL); assert(sBuffer != NULL);
assert(dict != NULL); assert(dict != NULL);
if ((size_t)workspace & 7) return NULL; /* 8-aligned */ if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
if (workspaceSize < neededSpace) return NULL; if (sBufferSize < neededSpace) return NULL;
if (dictLoadMethod == ZSTD_dlm_byCopy) { if (dictLoadMethod == ZSTD_dlm_byCopy) {
memcpy(ddict+1, dict, dictSize); /* local copy */ memcpy(ddict+1, dict, dictSize); /* local copy */
dict = ddict+1; dict = ddict+1;
} }
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, ZSTD_dlm_byRef, dictContentType) )) if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
dict, dictSize,
ZSTD_dlm_byRef, dictContentType) ))
return NULL; return NULL;
return ddict; return ddict;
} }