From 55fc1f91fd16d1ebddd4787788876fc14878abe6 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 24 May 2017 13:50:10 -0700 Subject: [PATCH 1/2] [zstd] Fix up formatting edge cases for clang-format --- lib/compress/huf_compress.c | 3 +- lib/compress/zstd_compress.c | 7 +-- lib/decompress/huf_decompress.c | 3 +- lib/decompress/zstd_decompress.c | 95 ++++++++++++++++---------------- 4 files changed, 55 insertions(+), 53 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 7248c2513..7af0789a9 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -266,7 +266,8 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) if (highTotal <= lowTotal) break; } } /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) nBitsToDecrease ++; totalCost -= 1 << (nBitsToDecrease-1); if (rankLast[nBitsToDecrease-1] == noSymbol) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index edf4609f7..3ba1748f0 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1950,8 +1950,8 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) } - -FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ +/* inlining is important to hardwire a hot branch (template emulation) */ +FORCE_INLINE size_t ZSTD_HcFindBestMatch_generic ( ZSTD_CCtx* zc, /* Index table will be updated */ const BYTE* const ip, const BYTE* const iLimit, @@ -2557,8 +2557,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); U32 const fcsCode = params.fParams.contentSizeFlag ? - (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */ - 0; + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); size_t pos; diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index ea35c3620..d40b3be04 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -545,7 +545,8 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { BIT_skipBits(DStream, dt[val].nbBits); if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) - DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); } } return 1; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 0c61960ec..ea64edeed 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -2240,7 +2240,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB return hSize; /* error */ } } #else - return hSize; + return hSize; #endif if (hSize != 0) { /* need more input */ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ @@ -2252,53 +2252,54 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; break; - } } - - /* check for single-pass mode opportunity */ - if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ - && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { - size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); - if (cSize <= (size_t)(iend-istart)) { - size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict); - if (ZSTD_isError(decompressedSize)) return decompressedSize; - ip = istart + cSize; - op += decompressedSize; - zds->dctx->expected = 0; - zds->stage = zdss_init; - someMoreWork = 0; - break; - } } - - /* Consume header */ - CHECK_F(ZSTD_decompressBegin_usingDDict(zds->dctx, zds->ddict)); - { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ - CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); - { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); - CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size)); - } } - - zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); - if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); - - /* Adapt buffer sizes to frame header instructions */ - { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); - size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2; - zds->blockSize = blockSize; - if (zds->inBuffSize < blockSize) { - ZSTD_free(zds->inBuff, zds->customMem); - zds->inBuffSize = 0; - zds->inBuff = (char*)ZSTD_malloc(blockSize, zds->customMem); - if (zds->inBuff == NULL) return ERROR(memory_allocation); - zds->inBuffSize = blockSize; } - if (zds->outBuffSize < neededOutSize) { - ZSTD_free(zds->outBuff, zds->customMem); - zds->outBuffSize = 0; - zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem); - if (zds->outBuff == NULL) return ERROR(memory_allocation); - zds->outBuffSize = neededOutSize; - } } - zds->stage = zdss_read; + + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); + if (cSize <= (size_t)(iend-istart)) { + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + ip = istart + cSize; + op += decompressedSize; + zds->dctx->expected = 0; + zds->stage = zdss_init; + someMoreWork = 0; + break; + } } + + /* Consume header */ + CHECK_F(ZSTD_decompressBegin_usingDDict(zds->dctx, zds->ddict)); + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); + { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size)); + } } + + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); + size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2; + zds->blockSize = blockSize; + if (zds->inBuffSize < blockSize) { + ZSTD_free(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->inBuff = (char*)ZSTD_malloc(blockSize, zds->customMem); + if (zds->inBuff == NULL) return ERROR(memory_allocation); + zds->inBuffSize = blockSize; + } + if (zds->outBuffSize < neededOutSize) { + ZSTD_free(zds->outBuff, zds->customMem); + zds->outBuffSize = 0; + zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem); + if (zds->outBuff == NULL) return ERROR(memory_allocation); + zds->outBuffSize = neededOutSize; + } } + zds->stage = zdss_read; + } /* pass-through */ case zdss_read: From 74b12f401455dcd50a8e858b7bb18acd9eecacc7 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 24 May 2017 13:52:36 -0700 Subject: [PATCH 2/2] [linux-kernel] Fix some up clang-format edge cases --- contrib/linux-kernel/kernelize.sh | 1 + contrib/linux-kernel/lib/zstd/compress.c | 5 ++--- contrib/linux-kernel/lib/zstd/decompress.c | 2 +- contrib/linux-kernel/lib/zstd/fse.h | 4 ---- contrib/linux-kernel/lib/zstd/fse_compress.c | 4 ---- contrib/linux-kernel/lib/zstd/fse_decompress.c | 4 ---- contrib/linux-kernel/lib/zstd/huf_compress.c | 5 ++--- contrib/linux-kernel/lib/zstd/huf_decompress.c | 5 ++--- 8 files changed, 8 insertions(+), 22 deletions(-) diff --git a/contrib/linux-kernel/kernelize.sh b/contrib/linux-kernel/kernelize.sh index 35ef5ed9f..21aa2ecdc 100755 --- a/contrib/linux-kernel/kernelize.sh +++ b/contrib/linux-kernel/kernelize.sh @@ -96,6 +96,7 @@ then check_not_present_in_file STATIC_ASSERT ${LIB}mem.h check_not_present_in_file "#define ZSTD_STATIC_ASSERT" ${LIB}compress.c check_not_present MEM_STATIC + check_not_present FSE_COMMONDEFS_ONLY check_not_present "#if 0" check_not_present "#if 1" check_not_present _MSC_VER diff --git a/contrib/linux-kernel/lib/zstd/compress.c b/contrib/linux-kernel/lib/zstd/compress.c index 39b8c3b8f..f2940a146 100644 --- a/contrib/linux-kernel/lib/zstd/compress.c +++ b/contrib/linux-kernel/lib/zstd/compress.c @@ -2412,9 +2412,8 @@ static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_paramete U32 const windowSize = 1U << params.cParams.windowLog; U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); - U32 const fcsCode = params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) - : /* 0-3 */ - 0; + U32 const fcsCode = + params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */ BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6)); size_t pos; diff --git a/contrib/linux-kernel/lib/zstd/decompress.c b/contrib/linux-kernel/lib/zstd/decompress.c index 2926b36fc..def10ea4d 100644 --- a/contrib/linux-kernel/lib/zstd/decompress.c +++ b/contrib/linux-kernel/lib/zstd/decompress.c @@ -2300,7 +2300,6 @@ size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inB ip += toLoad; break; } - } /* check for single-pass mode opportunity */ if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ @@ -2355,6 +2354,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inB } } zds->stage = zdss_read; + } /* pass-through */ case zdss_read: { diff --git a/contrib/linux-kernel/lib/zstd/fse.h b/contrib/linux-kernel/lib/zstd/fse.h index 9749755c3..bc2962aec 100644 --- a/contrib/linux-kernel/lib/zstd/fse.h +++ b/contrib/linux-kernel/lib/zstd/fse.h @@ -536,8 +536,6 @@ ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bi ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr) { return DStatePtr->state == 0; } -#ifndef FSE_COMMONDEFS_ONLY - /* ************************************************************** * Tuning parameters ****************************************************************/ @@ -567,8 +565,6 @@ ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr) { return DSt #define FSE_FUNCTION_EXTENSION #define FSE_DECODE_TYPE FSE_decode_t -#endif /* !FSE_COMMONDEFS_ONLY */ - /* *************************************************************** * Constants *****************************************************************/ diff --git a/contrib/linux-kernel/lib/zstd/fse_compress.c b/contrib/linux-kernel/lib/zstd/fse_compress.c index 6bb810ff8..e016bb177 100644 --- a/contrib/linux-kernel/lib/zstd/fse_compress.c +++ b/contrib/linux-kernel/lib/zstd/fse_compress.c @@ -180,8 +180,6 @@ size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsi return 0; } -#ifndef FSE_COMMONDEFS_ONLY - /*-************************************************************** * FSE NCount encoding-decoding ****************************************************************/ @@ -857,5 +855,3 @@ size_t FSE_compress_wksp(void *dst, size_t dstSize, const void *src, size_t srcS return op - ostart; } - -#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/contrib/linux-kernel/lib/zstd/fse_decompress.c b/contrib/linux-kernel/lib/zstd/fse_decompress.c index 245570a8a..96cf89ff9 100644 --- a/contrib/linux-kernel/lib/zstd/fse_decompress.c +++ b/contrib/linux-kernel/lib/zstd/fse_decompress.c @@ -161,8 +161,6 @@ size_t FSE_buildDTable(FSE_DTable *dt, const short *normalizedCounter, unsigned return 0; } -#ifndef FSE_COMMONDEFS_ONLY - /*-******************************************************* * Decompression (Byte symbols) *********************************************************/ @@ -313,5 +311,3 @@ size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size return FSE_decompress_usingDTable(dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ } - -#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/contrib/linux-kernel/lib/zstd/huf_compress.c b/contrib/linux-kernel/lib/zstd/huf_compress.c index 158740129..e82a136a1 100644 --- a/contrib/linux-kernel/lib/zstd/huf_compress.c +++ b/contrib/linux-kernel/lib/zstd/huf_compress.c @@ -305,9 +305,8 @@ static U32 HUF_setMaxHeight(nodeElt *huffNode, U32 lastNonNull, U32 maxNbBits) } } /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - while ( - (nBitsToDecrease <= HUF_TABLELOG_MAX) && - (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease <= HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) nBitsToDecrease++; totalCost -= 1 << (nBitsToDecrease - 1); if (rankLast[nBitsToDecrease - 1] == noSymbol) diff --git a/contrib/linux-kernel/lib/zstd/huf_decompress.c b/contrib/linux-kernel/lib/zstd/huf_decompress.c index 5c38aa30d..950c19443 100644 --- a/contrib/linux-kernel/lib/zstd/huf_decompress.c +++ b/contrib/linux-kernel/lib/zstd/huf_decompress.c @@ -573,9 +573,8 @@ static U32 HUF_decodeLastSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DE if (DStream->bitsConsumed < (sizeof(DStream->bitContainer) * 8)) { BIT_skipBits(DStream, dt[val].nbBits); if (DStream->bitsConsumed > (sizeof(DStream->bitContainer) * 8)) - DStream->bitsConsumed = - (sizeof(DStream->bitContainer) * - 8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer) * 8); } } return 1;