krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-07 09:26:03 +02:00
Code Issues Releases 1 Activity

Merge pull request #3460 from daniellerozenblit/fix-long-offsets-resolution-pointer

Browse Source 
fix long offset resolution
This commit is contained in:
daniellerozenblit 2023-01-30 14:02:51 -05:00 committed by GitHub
commit 00176638e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 87 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/dev-long-tests.yml vendored
View File

@ -65,6 +65,17 @@ jobs:
- name: thread sanitizer fuzztest
run: CC=clang make tsan-fuzztest
big-tests-zstreamtest32:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- name: zstream tests in 32bit mode, with big tests
run: |
sudo apt-get -qqq update
make libc6install
CC=clang make -C tests test-zstream32 FUZZER_FLAGS="--big-tests"
# lasts ~23mn
gcc-8-asan-ubsan-testzstd:
runs-on: ubuntu-latest

2
lib/common/zstd_internal.h
View File

@ -346,7 +346,7 @@ typedef struct {
} ZSTD_frameSizeInfo; /* decompress & legacy */
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
/* custom memory allocation functions */
void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);

18
lib/compress/zstd_compress.c
View File

@ -2559,7 +2559,7 @@ static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* par
/* See doc/zstd_compression_format.md for detailed format description */
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
int ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
{
const seqDef* const sequences = seqStorePtr->sequencesStart;
BYTE* const llCodeTable = seqStorePtr->llCode;
@ -2567,18 +2567,24 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
BYTE* const mlCodeTable = seqStorePtr->mlCode;
U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
U32 u;
int longOffsets = 0;
assert(nbSeq <= seqStorePtr->maxNbSeq);
for (u=0; u<nbSeq; u++) {
U32 const llv = sequences[u].litLength;
U32 const ofCode = ZSTD_highbit32(sequences[u].offBase);
U32 const mlv = sequences[u].mlBase;
llCodeTable[u] = (BYTE)ZSTD_LLcode(llv);
ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offBase);
ofCodeTable[u] = (BYTE)ofCode;
mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
assert(!(MEM_64bits() && ofCode >= STREAM_ACCUMULATOR_MIN));
if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN)
longOffsets = 1;
}
if (seqStorePtr->longLengthType==ZSTD_llt_literalLength)
llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
if (seqStorePtr->longLengthType==ZSTD_llt_matchLength)
mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
return longOffsets;
}
/* ZSTD_useTargetCBlockSize():
@ -2612,6 +2618,7 @@ typedef struct {
U32 MLtype;
size_t size;
size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
int longOffsets;
} ZSTD_symbolEncodingTypeStats_t;
/* ZSTD_buildSequencesStatistics():
@ -2642,7 +2649,7 @@ ZSTD_buildSequencesStatistics(
stats.lastCountSize = 0;
/* convert length/distances into codes */
ZSTD_seqToCodes(seqStorePtr);
stats.longOffsets = ZSTD_seqToCodes(seqStorePtr);
assert(op <= oend);
assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */
/* build CTable for Literal Lengths */
@ -2756,7 +2763,6 @@ ZSTD_entropyCompressSeqStore_internal(
void* entropyWorkspace, size_t entropyWkspSize,
const int bmi2)
{
const int longOffsets = cctxParams->cParams.windowLog >= STREAM_ACCUMULATOR_MIN;
ZSTD_strategy const strategy = cctxParams->cParams.strategy;
unsigned* count = (unsigned*)entropyWorkspace;
FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
@ -2771,6 +2777,7 @@ ZSTD_entropyCompressSeqStore_internal(
BYTE* const oend = ostart + dstCapacity;
BYTE* op = ostart;
size_t lastCountSize;
int longOffsets = 0;
entropyWorkspace = count + (MaxSeq + 1);
entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
@ -2832,6 +2839,7 @@ ZSTD_entropyCompressSeqStore_internal(
*seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2));
lastCountSize = stats.lastCountSize;
op += stats.size;
longOffsets = stats.longOffsets;
}
{ size_t const bitstreamSize = ZSTD_encodeSequences(
@ -3476,7 +3484,7 @@ ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
static ZSTD_symbolEncodingTypeStats_t
ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy)
{
ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0};
ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0};
nextEntropy->litlength_repeatMode = FSE_repeat_none;
nextEntropy->offcode_repeatMode = FSE_repeat_none;
nextEntropy->matchlength_repeatMode = FSE_repeat_none;

2
tests/Makefile
View File

@ -373,7 +373,7 @@ test-zstream: zstreamtest
$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
test-zstream32: zstreamtest32
$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
$(QEMU_SYS) ./zstreamtest32 -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
test-longmatch: longmatch
$(QEMU_SYS) ./longmatch

61
tests/zstreamtest.c
View File

@ -2222,6 +2222,66 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Testing large offset with small window size: ", testNb++);
{
ZSTD_CCtx* cctx = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
/* Test large offset, small window size*/
{
size_t srcSize = 21;
void* const src = CNBuffer;
size_t dstSize = ZSTD_compressBound(srcSize);
void* const dst = compressedBuffer;
size_t const kNbSequences = 4;
ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
void* const checkBuf = malloc(srcSize);
const size_t largeDictSize = 1 << 25;
ZSTD_CDict* cdict = NULL;
ZSTD_DDict* ddict = NULL;
/* Generate large dictionary */
void* dictBuffer = calloc(largeDictSize, 1);
ZSTD_compressionParameters cParams = ZSTD_getCParams(1, srcSize, largeDictSize);
cParams.minMatch = ZSTD_MINMATCH_MIN;
cParams.hashLog = ZSTD_HASHLOG_MIN;
cParams.chainLog = ZSTD_CHAINLOG_MIN;
cdict = ZSTD_createCDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
ddict = ZSTD_createDDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
ZSTD_CCtx_refCDict(cctx, cdict);
ZSTD_DCtx_refDDict(dctx, ddict);
sequences[0] = (ZSTD_Sequence) {3, 3, 3, 0};
sequences[1] = (ZSTD_Sequence) {1 << 25, 0, 3, 0};
sequences[2] = (ZSTD_Sequence) {1 << 25, 0, 9, 0};
sequences[3] = (ZSTD_Sequence) {3, 0, 3, 0};
cSize = ZSTD_compressSequences(cctx, dst, dstSize,
sequences, kNbSequences,
src, srcSize);
CHECK(ZSTD_isError(cSize), "Should not throw an error");
{
size_t dSize = ZSTD_decompressDCtx(dctx, checkBuf, srcSize, dst, cSize);
CHECK(ZSTD_isError(dSize), "Should not throw an error");
CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
}
free(sequences);
free(checkBuf);
free(dictBuffer);
ZSTD_freeCDict(cdict);
ZSTD_freeDDict(ddict);
}
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
_end:
FUZ_freeDictionary(dictionary);
ZSTD_freeCStream(zc);
@ -3034,6 +3094,7 @@ int main(int argc, const char** argv)
if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
if (!strcmp(argument, "--big-tests")) { bigTests=1; continue; }
argument++;
while (*argument!=0) {