krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-06 08:49:28 +02:00
Code Issues Releases 1 Activity

created ZSTD_entropyCompressSeqStore_wExtLitBuffer()

Browse Source 
can receive externally defined buffer of literals
This commit is contained in:
Yann Collet 2024-12-10 13:33:48 -08:00
parent e9f8a119b4
commit 0165eeb441
1 changed files with 47 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
lib/compress/zstd_compress.c
View File

@ -1039,7 +1039,7 @@ size_t ZSTD_CCtxParams_getParameter(
switch(param) switch(param)
{ {
case ZSTD_c_format : case ZSTD_c_format :
*value = CCtxParams->format; *value = (int)CCtxParams->format;
break; break;
case ZSTD_c_compressionLevel : case ZSTD_c_compressionLevel :
*value = CCtxParams->compressionLevel; *value = CCtxParams->compressionLevel;
@ -1054,16 +1054,16 @@ size_t ZSTD_CCtxParams_getParameter(
*value = (int)CCtxParams->cParams.chainLog; *value = (int)CCtxParams->cParams.chainLog;
break; break;
case ZSTD_c_searchLog : case ZSTD_c_searchLog :
*value = CCtxParams->cParams.searchLog; *value = (int)CCtxParams->cParams.searchLog;
break; break;
case ZSTD_c_minMatch : case ZSTD_c_minMatch :
*value = CCtxParams->cParams.minMatch; *value = (int)CCtxParams->cParams.minMatch;
break; break;
case ZSTD_c_targetLength : case ZSTD_c_targetLength :
*value = CCtxParams->cParams.targetLength; *value = (int)CCtxParams->cParams.targetLength;
break; break;
case ZSTD_c_strategy : case ZSTD_c_strategy :
*value = (unsigned)CCtxParams->cParams.strategy; *value = (int)CCtxParams->cParams.strategy;
break; break;
case ZSTD_c_contentSizeFlag : case ZSTD_c_contentSizeFlag :
*value = CCtxParams->fParams.contentSizeFlag; *value = CCtxParams->fParams.contentSizeFlag;
@ -1078,10 +1078,10 @@ size_t ZSTD_CCtxParams_getParameter(
*value = CCtxParams->forceWindow; *value = CCtxParams->forceWindow;
break; break;
case ZSTD_c_forceAttachDict : case ZSTD_c_forceAttachDict :
*value = CCtxParams->attachDictPref; *value = (int)CCtxParams->attachDictPref;
break; break;
case ZSTD_c_literalCompressionMode : case ZSTD_c_literalCompressionMode :
*value = CCtxParams->literalCompressionMode; *value = (int)CCtxParams->literalCompressionMode;
break; break;
case ZSTD_c_nbWorkers : case ZSTD_c_nbWorkers :
#ifndef ZSTD_MULTITHREAD #ifndef ZSTD_MULTITHREAD
@ -1115,19 +1115,19 @@ size_t ZSTD_CCtxParams_getParameter(
*value = CCtxParams->enableDedicatedDictSearch; *value = CCtxParams->enableDedicatedDictSearch;
break; break;
case ZSTD_c_enableLongDistanceMatching : case ZSTD_c_enableLongDistanceMatching :
*value = CCtxParams->ldmParams.enableLdm; *value = (int)CCtxParams->ldmParams.enableLdm;
break; break;
case ZSTD_c_ldmHashLog : case ZSTD_c_ldmHashLog :
*value = CCtxParams->ldmParams.hashLog; *value = (int)CCtxParams->ldmParams.hashLog;
break; break;
case ZSTD_c_ldmMinMatch : case ZSTD_c_ldmMinMatch :
*value = CCtxParams->ldmParams.minMatchLength; *value = (int)CCtxParams->ldmParams.minMatchLength;
break; break;
case ZSTD_c_ldmBucketSizeLog : case ZSTD_c_ldmBucketSizeLog :
*value = CCtxParams->ldmParams.bucketSizeLog; *value = (int)CCtxParams->ldmParams.bucketSizeLog;
break; break;
case ZSTD_c_ldmHashRateLog : case ZSTD_c_ldmHashRateLog :
*value = CCtxParams->ldmParams.hashRateLog; *value = (int)CCtxParams->ldmParams.hashRateLog;
break; break;
case ZSTD_c_targetCBlockSize : case ZSTD_c_targetCBlockSize :
*value = (int)CCtxParams->targetCBlockSize; *value = (int)CCtxParams->targetCBlockSize;
@ -1401,7 +1401,7 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog);
BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch);
BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength);
BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); BOUNDCHECK(ZSTD_c_strategy, (int)cParams.strategy);
return 0; return 0;
} }
@ -2558,7 +2558,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog) ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog)
: 0; : 0;
size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
int const h3log = srcCCtx->blockState.matchState.hashLog3; U32 const h3log = srcCCtx->blockState.matchState.hashLog3;
size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable,
@ -3011,19 +3011,20 @@ ZSTD_entropyCompressSeqStore_internal(
} }
static size_t static size_t
ZSTD_entropyCompressSeqStore( ZSTD_entropyCompressSeqStore_wExtLitBuffer(
void* dst, size_t dstCapacity,
const void* literals, size_t litSize,
size_t srcSize,
const SeqStore_t* seqStorePtr, const SeqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy, const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy, ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams, const ZSTD_CCtx_params* cctxParams,
void* dst, size_t dstCapacity,
size_t srcSize,
void* entropyWorkspace, size_t entropyWkspSize, void* entropyWorkspace, size_t entropyWkspSize,
int bmi2) int bmi2)
{ {
size_t const cSize = ZSTD_entropyCompressSeqStore_internal( size_t const cSize = ZSTD_entropyCompressSeqStore_internal(
dst, dstCapacity, dst, dstCapacity,
seqStorePtr->litStart, (size_t)(seqStorePtr->lit - seqStorePtr->litStart), literals, litSize,
seqStorePtr, prevEntropy, nextEntropy, cctxParams, seqStorePtr, prevEntropy, nextEntropy, cctxParams,
entropyWorkspace, entropyWkspSize, bmi2); entropyWorkspace, entropyWkspSize, bmi2);
if (cSize == 0) return 0; if (cSize == 0) return 0;
@ -3048,6 +3049,28 @@ ZSTD_entropyCompressSeqStore(
return cSize; return cSize;
} }
static size_t
ZSTD_entropyCompressSeqStore(
const SeqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
void* dst, size_t dstCapacity,
size_t srcSize,
void* entropyWorkspace, size_t entropyWkspSize,
int bmi2)
{
return ZSTD_entropyCompressSeqStore_wExtLitBuffer(
dst, dstCapacity,
seqStorePtr->litStart, (size_t)(seqStorePtr->lit - seqStorePtr->litStart),
srcSize,
seqStorePtr,
prevEntropy, nextEntropy,
cctxParams,
entropyWorkspace, entropyWkspSize,
bmi2);
}
/* ZSTD_selectBlockCompressor() : /* ZSTD_selectBlockCompressor() :
* Not static, but internal use only (used by long distance matcher) * Not static, but internal use only (used by long distance matcher)
* assumption : strat is a valid strategy */ * assumption : strat is a valid strategy */
@ -3101,7 +3124,7 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramS
ZSTD_blockCompressor selectedCompressor; ZSTD_blockCompressor selectedCompressor;
ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1);
assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat));
DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder); DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder);
if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) { if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) {
static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = { static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = {
@ -3385,7 +3408,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const SeqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM]) static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const SeqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM])
{ {
const SeqDef* inSeqs = seqStore->sequencesStart; const SeqDef* inSeqs = seqStore->sequencesStart;
const size_t nbInSequences = seqStore->sequences - inSeqs; const size_t nbInSequences = (size_t)(seqStore->sequences - inSeqs);
const size_t nbInLiterals = (size_t)(seqStore->lit - seqStore->litStart); const size_t nbInLiterals = (size_t)(seqStore->lit - seqStore->litStart);
ZSTD_Sequence* outSeqs = seqCollector->seqIndex == 0 ? seqCollector->seqStart : seqCollector->seqStart + seqCollector->seqIndex; ZSTD_Sequence* outSeqs = seqCollector->seqIndex == 0 ? seqCollector->seqStart : seqCollector->seqStart + seqCollector->seqIndex;
@ -6931,10 +6954,10 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
if (!cctx->isFirstBlock && if (!cctx->isFirstBlock &&
ZSTD_maybeRLE(&cctx->seqStore) && ZSTD_maybeRLE(&cctx->seqStore) &&
ZSTD_isRLE(ip, blockSize)) { ZSTD_isRLE(ip, blockSize)) {
/* We don't want to emit our first block as a RLE even if it qualifies because /* We don't want to emit our first block as RLE even if it qualifies because
* doing so will cause the decoder (cli only) to throw a "should consume all input error." * doing so will cause the decoder (cli <= v1.4.3 only) to throw an (invalid) error
* This is only an issue for zstd <= v1.4.3 * "should consume all input error."
*/ */
compressedSeqsSize = 1; compressedSeqsSize = 1;
} }