From 610c8b9e338466451b1e96ab5bcb9d88b6d3a1c1 Mon Sep 17 00:00:00 2001 From: Danielle Rozenblit Date: Thu, 9 Feb 2023 07:37:37 -0800 Subject: [PATCH] initial commit --- .circleci/config.yml | 3 +- .circleci/images/primary/Dockerfile | 2 +- .github/workflows/dev-long-tests.yml | 127 +-- .github/workflows/dev-short-tests.yml | 304 ++++-- .../workflows/publish-release-artifacts.yml | 2 +- .github/workflows/scorecards.yml | 8 +- CHANGELOG | 73 +- LICENSE | 8 +- Makefile | 6 +- README.md | 12 +- build/VS2010/libzstd-dll/libzstd-dll.rc | 4 +- build/VS2010/zstd/zstd.rc | 4 +- build/cmake/CMakeLists.txt | 24 +- .../AddZstdCompilationFlags.cmake | 42 +- build/cmake/contrib/CMakeLists.txt | 2 +- build/cmake/contrib/gen_html/CMakeLists.txt | 2 +- build/cmake/contrib/pzstd/CMakeLists.txt | 2 +- build/cmake/lib/CMakeLists.txt | 8 +- build/cmake/programs/CMakeLists.txt | 2 +- build/cmake/tests/CMakeLists.txt | 11 +- build/meson/contrib/pzstd/meson.build | 3 +- build/meson/lib/meson.build | 2 +- build/meson/meson.build | 4 +- build/meson/meson_options.txt | 2 +- build/meson/programs/meson.build | 3 +- build/meson/tests/meson.build | 55 +- build/single_file_libs/zstd-in.c | 2 +- build/single_file_libs/zstddeclib-in.c | 2 +- contrib/diagnose_corruption/Makefile | 2 +- .../diagnose_corruption/check_flipped_bits.c | 2 +- contrib/externalMatchfinder/.gitignore | 2 + contrib/externalMatchfinder/Makefile | 40 + contrib/externalMatchfinder/README.md | 14 + contrib/externalMatchfinder/main.c | 107 ++ contrib/externalMatchfinder/matchfinder.c | 80 ++ contrib/externalMatchfinder/matchfinder.h | 26 + contrib/freestanding_lib/freestanding.py | 2 +- contrib/gen_html/Makefile | 2 +- contrib/gen_html/gen_html.cpp | 2 +- contrib/largeNbDicts/Makefile | 2 +- contrib/largeNbDicts/largeNbDicts.c | 8 +- contrib/linux-kernel/Makefile | 8 +- contrib/linux-kernel/decompress_sources.h | 2 +- contrib/linux-kernel/linux.mk | 2 +- contrib/linux-kernel/linux_zstd.h | 2 +- contrib/linux-kernel/mem.h | 2 +- contrib/linux-kernel/test/Makefile | 2 +- .../test/include/linux/compiler.h | 2 +- .../linux-kernel/test/include/linux/errno.h | 2 +- .../linux-kernel/test/include/linux/kernel.h | 2 +- .../linux-kernel/test/include/linux/limits.h | 2 +- .../linux-kernel/test/include/linux/math64.h | 2 +- .../linux-kernel/test/include/linux/module.h | 2 +- .../linux-kernel/test/include/linux/printk.h | 2 +- .../linux-kernel/test/include/linux/stddef.h | 2 +- .../linux-kernel/test/include/linux/swab.h | 2 +- .../linux-kernel/test/include/linux/types.h | 2 +- .../linux-kernel/test/include/linux/xxhash.h | 4 +- contrib/linux-kernel/test/static_test.c | 2 +- contrib/linux-kernel/test/test.c | 8 +- contrib/linux-kernel/zstd_common_module.c | 2 +- contrib/linux-kernel/zstd_compress_module.c | 2 +- contrib/linux-kernel/zstd_decompress_module.c | 2 +- contrib/linux-kernel/zstd_deps.h | 2 +- contrib/match_finders/zstd_edist.c | 2 +- contrib/match_finders/zstd_edist.h | 2 +- contrib/pzstd/ErrorHolder.h | 2 +- contrib/pzstd/Logging.h | 2 +- contrib/pzstd/Makefile | 2 +- contrib/pzstd/Options.cpp | 2 +- contrib/pzstd/Options.h | 2 +- contrib/pzstd/Pzstd.cpp | 11 +- contrib/pzstd/Pzstd.h | 2 +- contrib/pzstd/README.md | 2 +- contrib/pzstd/SkippableFrame.cpp | 2 +- contrib/pzstd/SkippableFrame.h | 2 +- contrib/pzstd/main.cpp | 2 +- contrib/pzstd/test/OptionsTest.cpp | 2 +- contrib/pzstd/test/PzstdTest.cpp | 2 +- contrib/pzstd/test/RoundTrip.h | 2 +- contrib/pzstd/test/RoundTripTest.cpp | 2 +- contrib/pzstd/utils/Buffer.h | 2 +- contrib/pzstd/utils/FileSystem.h | 22 +- contrib/pzstd/utils/Likely.h | 2 +- contrib/pzstd/utils/Portability.h | 16 + contrib/pzstd/utils/Range.h | 4 +- contrib/pzstd/utils/ResourcePool.h | 2 +- contrib/pzstd/utils/ScopeGuard.h | 2 +- contrib/pzstd/utils/ThreadPool.h | 2 +- contrib/pzstd/utils/WorkQueue.h | 2 +- contrib/pzstd/utils/test/BufferTest.cpp | 2 +- contrib/pzstd/utils/test/RangeTest.cpp | 2 +- contrib/pzstd/utils/test/ResourcePoolTest.cpp | 2 +- contrib/pzstd/utils/test/ScopeGuardTest.cpp | 2 +- contrib/pzstd/utils/test/ThreadPoolTest.cpp | 2 +- contrib/pzstd/utils/test/WorkQueueTest.cpp | 2 +- contrib/recovery/Makefile | 2 +- contrib/recovery/recover_directory.c | 2 +- contrib/seekable_format/examples/Makefile | 2 +- .../examples/parallel_compression.c | 2 +- .../examples/parallel_processing.c | 2 +- .../examples/seekable_compression.c | 2 +- .../examples/seekable_decompression.c | 2 +- .../examples/seekable_decompression_mem.c | 2 +- contrib/seekable_format/tests/Makefile | 2 +- .../seekable_format/tests/seekable_tests.c | 34 + .../zstd_seekable_compression_format.md | 2 +- contrib/seekable_format/zstdseek_compress.c | 4 +- contrib/seekable_format/zstdseek_decompress.c | 8 +- contrib/seqBench/Makefile | 2 +- doc/README.md | 5 +- doc/educational_decoder/Makefile | 2 +- doc/educational_decoder/harness.c | 2 +- doc/educational_decoder/zstd_decompress.c | 4 +- doc/educational_decoder/zstd_decompress.h | 2 +- doc/zstd_compression_format.md | 34 +- doc/zstd_manual.html | 266 ++++- examples/Makefile | 2 +- examples/common.h | 2 +- examples/dictionary_compression.c | 2 +- examples/dictionary_decompression.c | 2 +- examples/multiple_simple_compression.c | 2 +- examples/multiple_streaming_compression.c | 2 +- examples/simple_compression.c | 2 +- examples/simple_decompression.c | 2 +- examples/streaming_compression.c | 2 +- examples/streaming_compression_thread_pool.c | 2 +- examples/streaming_decompression.c | 2 +- examples/streaming_memory_usage.c | 2 +- lib/Makefile | 2 +- lib/README.md | 7 + lib/common/bits.h | 2 +- lib/common/bitstream.h | 2 +- lib/common/compiler.h | 17 +- lib/common/cpu.h | 2 +- lib/common/debug.c | 2 +- lib/common/debug.h | 2 +- lib/common/entropy_common.c | 11 +- lib/common/error_private.c | 8 +- lib/common/error_private.h | 2 +- lib/common/fse.h | 84 +- lib/common/fse_decompress.c | 96 +- lib/common/huf.h | 234 ++--- lib/common/mem.h | 51 +- lib/common/pool.c | 8 +- lib/common/pool.h | 2 +- lib/common/portability_macros.h | 10 +- lib/common/threading.c | 82 +- lib/common/threading.h | 15 +- lib/common/xxhash.c | 4 +- lib/common/xxhash.h | 10 +- lib/common/zstd_common.c | 2 +- lib/common/zstd_deps.h | 2 +- lib/common/zstd_internal.h | 9 +- lib/common/zstd_trace.h | 2 +- lib/compress/clevels.h | 2 +- lib/compress/fse_compress.c | 122 +-- lib/compress/hist.c | 2 +- lib/compress/hist.h | 2 +- lib/compress/huf_compress.c | 206 ++-- lib/compress/zstd_compress.c | 975 ++++++++++++------ lib/compress/zstd_compress_internal.h | 85 +- lib/compress/zstd_compress_literals.c | 110 +- lib/compress/zstd_compress_literals.h | 24 +- lib/compress/zstd_compress_sequences.c | 2 +- lib/compress/zstd_compress_sequences.h | 2 +- lib/compress/zstd_compress_superblock.c | 10 +- lib/compress/zstd_compress_superblock.h | 2 +- lib/compress/zstd_cwksp.h | 6 +- lib/compress/zstd_double_fast.c | 2 +- lib/compress/zstd_double_fast.h | 2 +- lib/compress/zstd_fast.c | 2 +- lib/compress/zstd_fast.h | 2 +- lib/compress/zstd_lazy.c | 3 +- lib/compress/zstd_lazy.h | 6 +- lib/compress/zstd_ldm.c | 2 +- lib/compress/zstd_ldm.h | 2 +- lib/compress/zstd_ldm_geartab.h | 2 +- lib/compress/zstd_opt.c | 115 ++- lib/compress/zstd_opt.h | 2 +- lib/compress/zstdmt_compress.c | 2 +- lib/compress/zstdmt_compress.h | 2 +- lib/decompress/huf_decompress.c | 871 ++++++++-------- lib/decompress/huf_decompress_amd64.S | 28 +- lib/decompress/zstd_ddict.c | 3 +- lib/decompress/zstd_ddict.h | 2 +- lib/decompress/zstd_decompress.c | 134 ++- lib/decompress/zstd_decompress_block.c | 205 +++- lib/decompress/zstd_decompress_block.h | 2 +- lib/decompress/zstd_decompress_internal.h | 3 +- lib/deprecated/zbuff.h | 2 +- lib/deprecated/zbuff_common.c | 2 +- lib/deprecated/zbuff_compress.c | 2 +- lib/deprecated/zbuff_decompress.c | 4 +- lib/dictBuilder/cover.c | 31 +- lib/dictBuilder/cover.h | 2 +- lib/dictBuilder/fastcover.c | 2 +- lib/dictBuilder/zdict.c | 14 +- lib/dll/example/Makefile | 2 +- lib/legacy/zstd_legacy.h | 9 +- lib/legacy/zstd_v01.c | 63 +- lib/legacy/zstd_v01.h | 2 +- lib/legacy/zstd_v02.c | 81 +- lib/legacy/zstd_v02.h | 2 +- lib/legacy/zstd_v03.c | 85 +- lib/legacy/zstd_v03.h | 2 +- lib/legacy/zstd_v04.c | 80 +- lib/legacy/zstd_v04.h | 2 +- lib/legacy/zstd_v05.c | 94 +- lib/legacy/zstd_v05.h | 2 +- lib/legacy/zstd_v06.c | 103 +- lib/legacy/zstd_v06.h | 2 +- lib/legacy/zstd_v07.c | 91 +- lib/legacy/zstd_v07.h | 2 +- lib/libzstd.mk | 42 +- lib/libzstd.pc.in | 6 +- lib/module.modulemap | 16 +- lib/zdict.h | 80 +- lib/zstd.h | 384 ++++++- lib/zstd_errors.h | 34 +- programs/Makefile | 13 +- programs/README.md | 2 +- programs/benchfn.c | 8 +- programs/benchfn.h | 2 +- programs/benchzstd.c | 34 +- programs/benchzstd.h | 2 +- programs/datagen.c | 2 +- programs/datagen.h | 2 +- programs/dibio.c | 11 +- programs/dibio.h | 2 +- programs/fileio.c | 701 ++++++++----- programs/fileio.h | 4 +- programs/fileio_asyncio.c | 99 +- programs/fileio_asyncio.h | 24 +- programs/fileio_common.h | 16 +- programs/fileio_types.h | 2 +- programs/platform.h | 14 +- programs/timefn.c | 151 ++- programs/timefn.h | 64 +- programs/util.c | 237 ++++- programs/util.h | 23 +- programs/windres/verrsrc.h | 2 +- programs/windres/zstd.rc | 4 +- programs/zstd.1 | 115 ++- programs/zstd.1.md | 146 +-- programs/zstdcli.c | 327 +++--- programs/zstdcli_trace.c | 2 +- programs/zstdcli_trace.h | 2 +- programs/zstdgrep.1.md | 10 +- programs/zstdless.1.md | 6 +- tests/DEPRECATED-test-zstd-speed.py | 2 +- tests/Makefile | 6 +- tests/automated_benchmarking.py | 2 +- tests/bigdict.c | 2 +- tests/checkTag.c | 2 +- tests/check_size.py | 31 + tests/cli-tests/README.md | 10 + tests/cli-tests/basic/help.sh.stdout.glob | 41 +- tests/cli-tests/basic/memlimit.sh | 14 +- tests/cli-tests/basic/version.sh.stdout.glob | 4 +- tests/cli-tests/compression/adapt.sh | 8 + tests/cli-tests/compression/levels.sh | 48 +- .../compression/levels.sh.stderr.exact | 52 +- tests/cli-tests/compression/multi-threaded.sh | 16 +- .../multi-threaded.sh.stderr.exact | 10 + tests/cli-tests/compression/window-resize.sh | 9 + .../window-resize.sh.stderr.ignore | 0 .../compression/window-resize.sh.stdout.glob | 3 + .../pass-through.sh.stderr.exact | 1 + .../dictionaries/dictionary-mismatch.sh | 2 +- .../dictionary-mismatch.sh.stderr.exact | 3 +- .../file-stat/compress-file-to-file.sh | 9 + .../compress-file-to-file.sh.stderr.exact | 42 + .../file-stat/compress-file-to-stdout.sh | 8 + .../compress-file-to-stdout.sh.stderr.exact | 24 + .../file-stat/compress-stdin-to-file.sh | 8 + .../compress-stdin-to-file.sh.stderr.exact | 24 + .../file-stat/compress-stdin-to-stdout.sh | 8 + .../compress-stdin-to-stdout.sh.stderr.exact | 18 + .../file-stat/decompress-file-to-file.sh | 8 + .../decompress-file-to-file.sh.stderr.exact | 38 + .../file-stat/decompress-file-to-stdout.sh | 7 + .../decompress-file-to-stdout.sh.stderr.exact | 18 + .../file-stat/decompress-stdin-to-file.sh | 7 + .../decompress-stdin-to-file.sh.stderr.exact | 20 + .../file-stat/decompress-stdin-to-stdout.sh | 7 + ...decompress-stdin-to-stdout.sh.stderr.exact | 14 + tests/cli-tests/progress/no-progress.sh | 46 + .../progress/no-progress.sh.stderr.glob | 96 ++ tests/cli-tests/progress/progress.sh | 41 + .../progress/progress.sh.stderr.glob | 62 ++ tests/cli-tests/run.py | 23 +- tests/datagencli.c | 2 +- tests/decodecorpus.c | 44 +- tests/external_matchfinder.c | 140 +++ tests/external_matchfinder.h | 39 + tests/fullbench.c | 9 +- tests/fuzz/Makefile | 32 +- tests/fuzz/block_decompress.c | 2 +- tests/fuzz/block_round_trip.c | 2 +- tests/fuzz/decompress_dstSize_tooSmall.c | 2 +- tests/fuzz/dictionary_decompress.c | 2 +- tests/fuzz/dictionary_loader.c | 2 +- tests/fuzz/dictionary_round_trip.c | 2 +- tests/fuzz/dictionary_stream_round_trip.c | 2 +- tests/fuzz/fse_read_ncount.c | 2 +- tests/fuzz/fuzz.h | 5 +- tests/fuzz/fuzz.py | 2 +- tests/fuzz/fuzz_data_producer.c | 2 +- tests/fuzz/fuzz_data_producer.h | 2 +- tests/fuzz/fuzz_helpers.c | 2 +- tests/fuzz/fuzz_helpers.h | 2 +- tests/fuzz/huf_decompress.c | 20 +- tests/fuzz/huf_round_trip.c | 30 +- tests/fuzz/raw_dictionary_round_trip.c | 2 +- tests/fuzz/regression_driver.c | 2 +- tests/fuzz/seekable_roundtrip.c | 2 +- tests/fuzz/sequence_compression_api.c | 128 ++- tests/fuzz/simple_compress.c | 2 +- tests/fuzz/simple_decompress.c | 2 +- tests/fuzz/simple_round_trip.c | 46 +- tests/fuzz/stream_decompress.c | 10 +- tests/fuzz/stream_round_trip.c | 20 +- tests/fuzz/zstd_frame_info.c | 2 +- tests/fuzz/zstd_helpers.c | 40 +- tests/fuzz/zstd_helpers.h | 2 +- tests/fuzzer.c | 537 ++++++++-- tests/golden-decompression/empty-block.zst | Bin 11 -> 11 bytes tests/gzip/Makefile | 2 +- tests/gzip/gzip-env.sh | 2 +- tests/gzip/helin-segv.sh | 2 +- tests/gzip/help-version.sh | 2 +- tests/gzip/hufts.sh | 2 +- tests/gzip/init.sh | 2 +- tests/gzip/keep.sh | 2 +- tests/gzip/list.sh | 2 +- tests/gzip/memcpy-abuse.sh | 2 +- tests/gzip/mixed.sh | 2 +- tests/gzip/null-suffix-clobber.sh | 2 +- tests/gzip/stdin.sh | 2 +- tests/gzip/test-driver.sh | 2 +- tests/gzip/trailing-nul.sh | 2 +- tests/gzip/unpack-invalid.sh | 2 +- tests/gzip/z-suffix.sh | 2 +- tests/gzip/zdiff.sh | 2 +- tests/gzip/zgrep-context.sh | 2 +- tests/gzip/zgrep-f.sh | 2 +- tests/gzip/zgrep-signal.sh | 2 +- tests/gzip/znew-k.sh | 2 +- tests/invalidDictionaries.c | 2 +- tests/legacy.c | 2 +- tests/longmatch.c | 35 +- tests/paramgrill.c | 40 +- tests/playTests.sh | 67 +- tests/poolTests.c | 2 +- tests/rateLimiter.py | 2 +- tests/regression/Makefile | 2 +- tests/regression/config.c | 2 +- tests/regression/config.h | 2 +- tests/regression/data.c | 2 +- tests/regression/data.h | 2 +- tests/regression/levels.h | 2 +- tests/regression/method.c | 2 +- tests/regression/method.h | 2 +- tests/regression/result.c | 2 +- tests/regression/result.h | 2 +- tests/regression/results.csv | 190 ++-- tests/regression/test.c | 2 +- tests/roundTripCrash.c | 2 +- tests/seqgen.c | 2 +- tests/seqgen.h | 2 +- tests/test-license.py | 6 +- tests/test-zstd-versions.py | 161 +-- tests/zstreamtest.c | 542 +++++++++- zlibWrapper/README.md | 4 +- zlibWrapper/examples/minigzip.c | 2 +- zlibWrapper/examples/zwrapbench.c | 18 +- zlibWrapper/gzclose.c | 2 +- zlibWrapper/gzcompatibility.h | 2 +- zlibWrapper/gzguts.h | 2 +- zlibWrapper/gzlib.c | 2 +- zlibWrapper/gzread.c | 2 +- zlibWrapper/gzwrite.c | 2 +- zlibWrapper/zstd_zlibwrapper.c | 8 +- zlibWrapper/zstd_zlibwrapper.h | 2 +- 385 files changed, 7522 insertions(+), 4043 deletions(-) create mode 100644 contrib/externalMatchfinder/.gitignore create mode 100644 contrib/externalMatchfinder/Makefile create mode 100644 contrib/externalMatchfinder/README.md create mode 100644 contrib/externalMatchfinder/main.c create mode 100644 contrib/externalMatchfinder/matchfinder.c create mode 100644 contrib/externalMatchfinder/matchfinder.h create mode 100644 contrib/pzstd/utils/Portability.h create mode 100755 tests/check_size.py create mode 100755 tests/cli-tests/compression/window-resize.sh create mode 100644 tests/cli-tests/compression/window-resize.sh.stderr.ignore create mode 100644 tests/cli-tests/compression/window-resize.sh.stdout.glob create mode 100755 tests/cli-tests/file-stat/compress-file-to-file.sh create mode 100644 tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact create mode 100755 tests/cli-tests/file-stat/compress-file-to-stdout.sh create mode 100644 tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact create mode 100755 tests/cli-tests/file-stat/compress-stdin-to-file.sh create mode 100644 tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact create mode 100755 tests/cli-tests/file-stat/compress-stdin-to-stdout.sh create mode 100644 tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact create mode 100755 tests/cli-tests/file-stat/decompress-file-to-file.sh create mode 100644 tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact create mode 100755 tests/cli-tests/file-stat/decompress-file-to-stdout.sh create mode 100644 tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact create mode 100755 tests/cli-tests/file-stat/decompress-stdin-to-file.sh create mode 100644 tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact create mode 100755 tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh create mode 100644 tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact create mode 100755 tests/cli-tests/progress/no-progress.sh create mode 100644 tests/cli-tests/progress/no-progress.sh.stderr.glob create mode 100755 tests/cli-tests/progress/progress.sh create mode 100644 tests/cli-tests/progress/progress.sh.stderr.glob create mode 100644 tests/external_matchfinder.c create mode 100644 tests/external_matchfinder.h diff --git a/.circleci/config.yml b/.circleci/config.yml index f2eab6807..621ac49ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,8 +7,6 @@ jobs: # preinstalled to reduce installation time. docker: - image: fbopensource/zstd-circleci-primary:0.0.1 - # TODO: Re-enable aarch64 build: - # make aarch64build && make clean steps: - checkout - run: @@ -36,6 +34,7 @@ jobs: make ppc64build V=1; make clean make ppcbuild V=1; make clean make armbuild V=1; make clean + make aarch64build V=1; make clean make -C tests test-legacy test-longmatch; make clean make -C lib libzstd-nomt; make clean # This step should only be run in a cron job diff --git a/.circleci/images/primary/Dockerfile b/.circleci/images/primary/Dockerfile index dd8004152..bfcb6af69 100644 --- a/.circleci/images/primary/Dockerfile +++ b/.circleci/images/primary/Dockerfile @@ -1,4 +1,4 @@ -FROM circleci/buildpack-deps:bionic +FROM circleci/buildpack-deps:focal RUN sudo dpkg --add-architecture i386 RUN sudo apt-get -y -qq update diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml index ac34c7165..1c8c9ec55 100644 --- a/.github/workflows/dev-long-tests.yml +++ b/.github/workflows/dev-long-tests.yml @@ -13,7 +13,7 @@ jobs: make-all: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: make all run: make all @@ -24,7 +24,7 @@ jobs: DEVNULLRIGHTS: 1 READFROMBLOCKDEVICE: 1 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: make test run: make test @@ -32,28 +32,28 @@ jobs: make-test-osx: runs-on: macos-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: OS-X test run: make test # make -c lib all doesn't work because of the fact that it's not a tty no-intrinsics-fuzztest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: no intrinsics fuzztest run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest tsan-zstreamtest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: thread sanitizer zstreamtest run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream ubsan-zstreamtest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: undefined behavior sanitizer zstreamtest run: CC=clang make uasan-test-zstream @@ -61,15 +61,26 @@ jobs: tsan-fuzztest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: thread sanitizer fuzztest run: CC=clang make tsan-fuzztest + + big-tests-zstreamtest32: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 + - name: zstream tests in 32bit mode, with big tests + run: | + sudo apt-get -qqq update + make libc6install + CC=clang make -C tests test-zstream32 FUZZER_FLAGS="--big-tests" + # lasts ~23mn gcc-8-asan-ubsan-testzstd: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: gcc-8 + ASan + UBSan + Test Zstd # See https://askubuntu.com/a/1428822 run: | @@ -81,14 +92,14 @@ jobs: clang-asan-ubsan-testzstd: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: clang + ASan + UBSan + Test Zstd run: CC=clang make -j uasan-test-zstd - msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v142 + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=${{matrix.toolset}} /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}} -# TODO: fix as part of https://github.com/facebook/zstd/issues/3064 -# visual-2015: -# # only GH actions windows-2016 contains VS 2015 -# runs-on: windows-2016 -# strategy: -# matrix: -# platform: [x64, Win32] -# configuration: [Debug, Release] -# steps: -# - uses: actions/checkout@v3 -# - name: Add MSBuild to PATH -# uses: microsoft/setup-msbuild@v1.1.3 -# - name: Build -# working-directory: ${{env.GITHUB_WORKSPACE}} -# run: > -# msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 -# /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}} + # This tests that we don't accidently grow the size too much. + # If the size grows intentionally, you can raise these numbers. + # But we do need to think about binary size, since it is a concern. + libzstd-size: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 + - name: libzstd size test + run: | + make clean && make -j -C lib libzstd && ./tests/check_size.py lib/libzstd.so 1100000 + make clean && make -j -C lib libzstd ZSTD_LIB_COMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 && ./tests/check_size.py lib/libzstd.so 400000 + make clean && make -j -C lib libzstd ZSTD_LIB_MINIFY=1 && ./tests/check_size.py lib/libzstd.so 300000 + make clean && make -j -C lib libzstd ZSTD_LIB_MINIFY=1 ZSTD_LIB_COMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 && ./tests/check_size.py lib/libzstd.so 80000 minimal-decompressor-macros: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: minimal decompressor macros run: | make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror" @@ -254,7 +310,7 @@ jobs: dynamic-bmi2: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: dynamic bmi2 tests run: | make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2" @@ -266,7 +322,7 @@ jobs: test-variants: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: make all variants & validate run: | make -j -C programs allVariants MOREFLAGS=-O0 @@ -275,7 +331,7 @@ jobs: qemu-consistency: name: QEMU ${{ matrix.name }} - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed. matrix: @@ -292,7 +348,7 @@ jobs: XCC: ${{ matrix.xcc }} XEMU: ${{ matrix.xemu }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: apt update & install run: | sudo apt-get update @@ -335,52 +391,58 @@ jobs: LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check mingw-short-test: - runs-on: windows-2019 + runs-on: windows-latest strategy: - fail-fast: false + fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed. matrix: include: [ - { compiler: gcc, platform: x64, script: "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"}, - { compiler: gcc, platform: x86, script: "CFLAGS=-Werror make -j allzstd"}, - { compiler: clang, platform: x64, script: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"}, + { compiler: gcc, msystem: MINGW32, cflags: "-Werror"}, + { compiler: gcc, msystem: MINGW64, cflags: "-Werror"}, + { compiler: clang, msystem: MINGW64, cflags: "--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion -Wno-unused-command-line-argument"}, ] + defaults: + run: + shell: msys2 {0} steps: - - uses: actions/checkout@v3 - - name: Mingw short test + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 + - uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.msystem }} + install: make diffutils + update: true + # Based on https://ariya.io/2020/07/on-github-actions-with-msys2 + - name: install mingw gcc i686 + if: ${{ (matrix.msystem == 'MINGW32') && (matrix.compiler == 'gcc') }} + run: pacman --noconfirm -S mingw-w64-i686-gcc + - name: install mingw gcc x86_64 + if: ${{ (matrix.msystem == 'MINGW64') && (matrix.compiler == 'gcc') }} + run: pacman --noconfirm -S mingw-w64-x86_64-gcc + - name: install mingw clang i686 + if: ${{ (matrix.msystem == 'MINGW32') && (matrix.compiler == 'clang') }} + run: pacman --noconfirm -S mingw-w64-i686-clang + - name: install mingw clang x86_64 + if: ${{ (matrix.msystem == 'MINGW64') && (matrix.compiler == 'clang') }} + run: pacman --noconfirm -S mingw-w64-x86_64-clang + - name: run mingw tests run: | - ECHO "Building ${{matrix.compiler}} ${{matrix.platform}}" - $env:PATH_ORIGINAL = $env:PATH - $env:PATH_MINGW32 = "C:\msys64\mingw32\bin" - $env:PATH_MINGW64 = "C:\msys64\mingw64\bin" - COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw32\bin\make.exe - COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw64\bin\make.exe - IF ("${{matrix.platform}}" -eq "x64") - { - $env:PATH = $env:PATH_MINGW64 + ";" + $env:PATH_ORIGINAL - } - ELSEIF ("${{matrix.platform}}" -eq "x86") - { - $env:PATH = $env:PATH_MINGW32 + ";" + $env:PATH_ORIGINAL - } make -v - sh -c "${{matrix.compiler}} -v" - $env:CC = "${{matrix.compiler}}" - sh -c "${{matrix.script}}" - ECHO "Testing ${{matrix.compiler}} ${{matrix.platform}}" + export CC=${{ matrix.compiler }} + $CC --version + CFLAGS="${{ matrix.cflags }}" make -j allzstd + echo "Testing $CC ${{ matrix.msystem }}" make clean - make check - + MSYS="" make check visual-runtime-tests: - runs-on: windows-2019 + runs-on: windows-latest strategy: matrix: platform: [x64, Win32] configuration: [Release] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: Add MSBuild to PATH - uses: microsoft/setup-msbuild@v1.1.3 + uses: microsoft/setup-msbuild@v1.3 - name: Build and run tests working-directory: ${{env.GITHUB_WORKSPACE}} env: @@ -392,13 +454,39 @@ jobs: COPY build\VS2010\bin\${{matrix.platform}}_${{matrix.configuration}}\*.exe tests\ CD tests sh -e playTests.sh - DIR .\fuzzer.exe -T2m + # Following instructions at: https://github.com/marketplace/actions/install-cygwin-action + cygwin-tests: + runs-on: windows-latest + steps: + - run: git config --global core.autocrlf input + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 + - uses: cygwin/cygwin-install-action@f5e0f048310c425e84bc789f493a828c6dc80a25 # tag=master + with: + platform: x86_64 + packages: >- + autoconf, + automake, + gcc-g++, + make, + mingw64-x86_64-gcc-g++, + patch, + perl + - name: cygwin tests + shell: C:\cygwin\bin\bash.exe --noprofile --norc -eo pipefail '{0}' + run: >- + export PATH=/usr/bin:$(cygpath ${SYSTEMROOT})/system32 && + export CFLAGS="-Werror -O1" && + ls && + make -j allzstd && + make -C tests fuzzer && + ./tests/fuzzer.exe -v -T1m + intel-cet-compatibility: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: Build Zstd run: | make -j zstd V=1 @@ -419,7 +507,7 @@ jobs: container: image: debian:testing steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: Install dependencies run: | apt -y update @@ -431,18 +519,44 @@ jobs: cc -Wall -Wextra -Wpedantic -Werror -o simple examples/simple_compression.c $(pkg-config --cflags --libs libzstd) ./simple LICENSE + versions-compatibility: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 + - name: Versions Compatibility Test + run: | + make -C tests versionsTest -# This test currently fails on Github Actions specifically. -# Possible reason : TTY emulation. -# Note that the same test works fine locally and on travisCI. -# This will have to be fixed before transferring the test to GA. -# versions-compatibility: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v3 -# - name: Versions Compatibility Test -# run: | -# make -C tests versionsTest + clangbuild: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 + - name: make clangbuild + run: | + make clangbuild + + clang-pgo: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 + - name: Build PGO Zstd with Clang + env: + CC: clang-14 + LLVM_PROFDATA: llvm-profdata-14 + run: | + make -C programs zstd-pgo + ./programs/zstd -b + + gcc-pgo: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 + - name: Build PGO Zstd with GCC + env: + CC: gcc + run: | + make -C programs zstd-pgo + ./programs/zstd -b # For reference : icc tests @@ -463,7 +577,7 @@ jobs: # sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" # sudo apt-get update # sudo apt-get install -y intel-basekit intel-hpckit -# - uses: actions/checkout@v3 +# - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 # - name: make check # run: | # make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check diff --git a/.github/workflows/publish-release-artifacts.yml b/.github/workflows/publish-release-artifacts.yml index 4a70fb8c6..2c89a91a5 100644 --- a/.github/workflows/publish-release-artifacts.yml +++ b/.github/workflows/publish-release-artifacts.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 - name: Archive env: diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 1c19b48b2..6a9e5217e 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -27,12 +27,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.0.0 + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # tag=v2.0.6 + uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # tag=v2.1.2 with: results_file: results.sarif results_format: sarif @@ -51,7 +51,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 # tag=v3.0.0 + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # tag=v3.1.2 with: name: SARIF file path: results.sarif @@ -59,6 +59,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@5f532563584d71fdef14ee64d17bafb34f751ce5 # tag=v1.0.26 + uses: github/codeql-action/upload-sarif@3ebbd71c74ef574dbc558c82f70e52732c8b44fe # tag=v2.2.1 with: sarif_file: results.sarif diff --git a/CHANGELOG b/CHANGELOG index e8c40f302..11cc0d3fb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,37 +1,62 @@ -v1.5.3 (June, 2022) -perf: 5-30% faster dictionary compression at levels 1-4 (#3086, #3114, #3152, @embg) -perf: 5-10% faster streaming compression at levels 1-2 (#3114, @embg) -perf: Remove branch in ZSTD_fast_noDict (#3129, @felixhandte) -perf: Add option to prefetch CDict tables (#3177, @embg) -perf: Minor compression ratio improvement (#2983, @Cyan4973) -perf: Minor speed improvement for Huffman decoding (#3013, @WojciechMula) -perf: Enable STATIC_BMI2 for gcc/clang (#3080, @TocarIP) -perf: Optimize ZSTD_row_getMatchMask for ARM levels 8-10 (#3139, #3160, @danlark1) -perf: aarch64 performance improvements (#3145, #3141, @JunHe77) -perf: Lazy parameters adaptation (#2974, @Cyan4973) -cli: Async write for decompression (#2975, @yoniko) -cli: Use buffered output (#2985, @yoniko) -cli: Change zstdless behavior to align with zless (#2909, @binhdvo) -cli: AsyncIO compression (#3021, #3022, @yoniko) -cli: Print zlib/lz4/lzma library versions in verbose version output (#3030, @terrelln) -cli: Fix for -r on empty directory (#3027, @brailovich) -cli: Fix required decompression memory usage reported by -vv + --long (#3042, @u1f35c) +v1.5.4 (Feb 2023) +perf: +20% faster huffman decompression for targets that can't compile x64 assembly (#3449, @terrelln) +perf: up to +10% faster streaming compression at levels 1-2 (#3114, @embg) +perf: +4-13% for levels 5-12 by optimizing function generation (#3295, @terrelln) +pref: +3-11% compression speed for `arm` target (#3199, #3164, #3145, #3141, #3138, @JunHe77 and #3139, #3160, @danlark1) +perf: +5-30% faster dictionary compression at levels 1-4 (#3086, #3114, #3152, @embg) +perf: +10-20% cold dict compression speed by prefetching CDict tables (#3177, @embg) +perf: +1% faster compression by removing a branch in ZSTD_fast_noDict (#3129, @felixhandte) +perf: Small compression ratio improvements in high compression mode (#2983, #3391, @Cyan4973 and #3285, #3302, @daniellerozenblit) +perf: small speed improvement by better detecting `STATIC_BMI2` for `clang` (#3080, @TocarIP) +perf: Improved streaming performance when `ZSTD_c_stableInBuffer` is set (#2974, @Cyan4973) +cli: Asynchronous I/O for improved cli speed (#2975, #2985, #3021, #3022, @yoniko) +cli: Change `zstdless` behavior to align with `zless` (#2909, @binhdvo) +cli: Keep original file if `-c` or `--stdout` is given (#3052, @dirkmueller) +cli: Keep original files when result is concatenated into a single output with `-o` (#3450, @Cyan4973) +cli: Preserve Permissions and Ownership of regular files (#3432, @felixhandte) +cli: Print zlib/lz4/lzma library versions with `-vv` (#3030, @terrelln) +cli: Print checksum value for single frame files with `-lv` (#3332, @Cyan4973) +cli: Print `dictID` when present with `-lv` (#3184, @htnhan) +cli: when `stderr` is *not* the console, disable status updates, but preserve final summary (#3458, @Cyan4973) +cli: support `--best` and `--no-name` in `gzip` compatibility mode (#3059, @dirkmueller) +cli: support for `posix` high resolution timer `clock_gettime()`, for improved benchmark accuracy (#3423, @Cyan4973) +cli: improved help/usage (`-h`, `-H`) formatting (#3094, @dirkmueller and #3385, @jonpalmisc) +cli: Fix better handling of bogus numeric values (#3268, @ctkhanhly) +cli: Fix input consists of multiple files _and_ `stdin` (#3222, @yoniko) +cli: Fix tiny files passthrough (#3215, @cgbur) +cli: Fix for `-r` on empty directory (#3027, @brailovich) +cli: Fix empty string as argument for `--output-dir-*` (#3220, @embg) +cli: Fix decompression memory usage reported by `-vv --long` (#3042, @u1f35c, and #3232, @zengyijing) cli: Fix infinite loop when empty input is passed to trainer (#3081, @terrelln) -cli: Implement more gzip compatibility (#3059, @dirkmueller) -cli: Keep original file if -c or --stdout is given (#3052, @dirkmueller) +cli: Fix `--adapt` doesn't work when `--no-progress` is also set (#3354, @terrelln) +api: Support for External matchfinder (#3333, @embg) +api: Support for in-place decompression (#3432, @terrelln) +api: New `ZSTD_CCtx_setCParams()` function, set all parameters defined in a `ZSTD_compressionParameters` structure (#3403, @Cyan4973) +api: Streaming decompression detects incorrect header ID sooner (#3175, @Cyan4973) +api: Window size resizing optimization for edge case (#3345, @daniellerozenblit) +api: More accurate error codes for busy-loop scenarios (#3413, #3455, @Cyan4973) +api: Fix limit overflow in `compressBound` and `decompressBound` (#3362, #3373, Cyan4973) reported by @nigeltao +api: Deprecate several advanced experimental functions: streaming (#3408, @embg), copy (#3196, @mileshu) +bug: Fix corruption that rarely occurs in 32-bit mode with wlog=25 (#3361, @terrelln) bug: Fix for block-splitter (#3033, @Cyan4973) bug: Fixes for Sequence Compression API (#3023, #3040, @Cyan4973) bug: Fix leaking thread handles on Windows (#3147, @animalize) bug: Fix timing issues with cmake/meson builds (#3166, #3167, #3170, @Cyan4973) build: Allow user to select legacy level for cmake (#3050, @shadchin) build: Enable legacy support by default in cmake (#3079, @niamster) -build: Meson improvements (#3039, #3122, @eli-schwartz) +build: Meson build script improvements (#3039, #3120, #3122, #3327, #3357, @eli-schwartz and #3276, @neheb) build: Add aarch64 to supported architectures for zstd_trace (#3054, @ooosssososos) -doc: Split help in long and short version, cleanup formatting (#3094, @dirkmueller) -doc: Updated man page, providing more details for --train mode (#3112, @Cyan4973) +build: support AIX architecture (#3219, @qiongsiwu) +build: Fix `ZSTD_LIB_MINIFY` build macro, which now reduces static library size by half (#3366, @terrelln) +build: Fix Windows issues with Multithreading translation layer (#3364, #3380, @yoniko) and ARM64 target (#3320, @cwoffenden) +build: Fix `cmake` script (#3382, #3392, @terrelln and #3252 @Tachi107 and #3167 @Cyan4973) +doc: Updated man page, providing more details for `--train` mode (#3112, @Cyan4973) doc: Add decompressor errata document (#3092, @terrelln) misc: Enable Intel CET (#2992, #2994, @hjl-tools) -misc: Streaming decompression can detect incorrect header ID sooner (#3175, @Cyan4973) +misc: Fix `contrib/` seekable format (#3058, @yhoogstrate and #3346, @daniellerozenblit) +misc: Improve speed of the one-file library generator (#3241, @wahern and #3005, @cwoffenden) + +v1.5.3 (dev version, unpublished) v1.5.2 (Jan, 2022) perf: Regain Minimal memset()-ing During Reuse of Compression Contexts (@Cyan4973, #2969) diff --git a/LICENSE b/LICENSE index a793a8028..75800288c 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,7 @@ BSD License For Zstandard software -Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -14,9 +14,9 @@ are permitted provided that the following conditions are met: this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED diff --git a/Makefile b/Makefile index 429c90ff1..75a47a308 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the @@ -123,6 +123,7 @@ contrib: lib $(MAKE) -C contrib/seekable_format/examples all $(MAKE) -C contrib/seekable_format/tests test $(MAKE) -C contrib/largeNbDicts all + $(MAKE) -C contrib/externalMatchfinder all cd build/single_file_libs/ ; ./build_decoder_test.sh cd build/single_file_libs/ ; ./build_library_test.sh @@ -142,6 +143,7 @@ clean: $(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID) $(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID) $(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID) + $(Q)$(MAKE) -C contrib/externalMatchfinder $@ > $(VOID) $(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp* $(Q)$(RM) -r lz4 @echo Cleaning completed @@ -402,7 +404,7 @@ cmakebuild: c89build: clean $(CC) -v - CFLAGS="-std=c89 -Werror -O0" $(MAKE) allmost # will fail, due to missing support for `long long` + CFLAGS="-std=c89 -Werror -Wno-attributes -Wpedantic -Wno-long-long -Wno-variadic-macros -O0" $(MAKE) lib zstd gnu90build: clean $(CC) -v diff --git a/README.md b/README.md index da8622d72..d34bda576 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Zstandard's format is stable and documented in [RFC8878](https://datatracker.iet This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library, and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files. Should your project require another programming language, -a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages). +a list of known ports and bindings is provided on [Zstandard homepage](https://facebook.github.io/zstd/#other-languages). **Development branch status:** @@ -39,7 +39,7 @@ compiled with [gcc] 9.3.0, on the [Silesia compression corpus]. [lzbench]: https://github.com/inikep/lzbench -[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia +[Silesia compression corpus]: https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia [gcc]: https://gcc.gnu.org/ | Compressor name | Ratio | Compression| Decompress.| @@ -56,8 +56,8 @@ on the [Silesia compression corpus]. | lzf 3.6 -1 | 2.077 | 410 MB/s | 830 MB/s | | snappy 1.1.9 | 2.073 | 550 MB/s | 1750 MB/s | -[zlib]: http://www.zlib.net/ -[lz4]: http://www.lz4.org/ +[zlib]: https://www.zlib.net/ +[lz4]: https://lz4.github.io/lz4/ The negative compression levels, specified with `--fast=#`, offer faster compression and decompression speed @@ -142,6 +142,10 @@ Other available options include: The `Makefile` follows the [GNU Standard Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html), allowing staged install, standard flags, directory variables and command variables. +For advanced use cases, specialized compilation flags which control binary generation +are documented in [`lib/README.md`](lib/README.md#modular-build) for the `libzstd` library +and in [`programs/README.md`](programs/README.md#compilation-variables) for the `zstd` CLI. + ### cmake A `cmake` project generator is provided within `build/cmake`. diff --git a/build/VS2010/libzstd-dll/libzstd-dll.rc b/build/VS2010/libzstd-dll/libzstd-dll.rc index ee9f56280..13e8746ff 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.rc +++ b/build/VS2010/libzstd-dll/libzstd-dll.rc @@ -32,11 +32,11 @@ BEGIN BEGIN BLOCK "040904B0" BEGIN - VALUE "CompanyName", "Yann Collet, Facebook, Inc." + VALUE "CompanyName", "Meta Platforms, Inc." VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm" VALUE "FileVersion", ZSTD_VERSION_STRING VALUE "InternalName", "libzstd.dll" - VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc." + VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates." VALUE "OriginalFilename", "libzstd.dll" VALUE "ProductName", "Zstandard" VALUE "ProductVersion", ZSTD_VERSION_STRING diff --git a/build/VS2010/zstd/zstd.rc b/build/VS2010/zstd/zstd.rc index f5e404730..a2118c2df 100644 --- a/build/VS2010/zstd/zstd.rc +++ b/build/VS2010/zstd/zstd.rc @@ -32,11 +32,11 @@ BEGIN BEGIN BLOCK "040904B0" BEGIN - VALUE "CompanyName", "Yann Collet, Facebook, Inc." + VALUE "CompanyName", "Meta Platforms, Inc." VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm" VALUE "FileVersion", ZSTD_VERSION_STRING VALUE "InternalName", "zstd.exe" - VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc." + VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates." VALUE "OriginalFilename", "zstd.exe" VALUE "ProductName", "Zstandard" VALUE "ProductVersion", ZSTD_VERSION_STRING diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index 5e2cf3d56..0bffc87d9 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the @@ -8,18 +8,18 @@ # ################################################################ cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) - -# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies. -# Set and use the newest cmake policies that are validated to work -set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3") + +# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies. +# Set and use the newest cmake policies that are validated to work +set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3") set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level if("${CMAKE_MAJOR_VERSION}" LESS 3) - set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") -elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND + set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") +elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND "${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}") - set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") -else() - set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0") + set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") +else() + set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0") endif() cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION}) @@ -40,16 +40,18 @@ if( CMAKE_MAJOR_VERSION LESS 3 ) set(PROJECT_VERSION_PATCH ${zstd_VERSION_PATCH}) set(PROJECT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}") enable_language(C) # Main library is in C + enable_language(ASM) # And ASM enable_language(CXX) # Testing contributed code also utilizes CXX else() project(zstd VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}" LANGUAGES C # Main library is in C + ASM # And ASM CXX # Testing contributed code also utilizes CXX ) endif() message(STATUS "ZSTD VERSION: ${zstd_VERSION}") -set(zstd_HOMEPAGE_URL "http://www.zstd.net") +set(zstd_HOMEPAGE_URL "https://facebook.github.io/zstd") set(zstd_DESCRIPTION "Zstandard is a real-time compression algorithm, providing high compression ratios.") # Set a default build type if none was specified diff --git a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake index 8d04458c3..0265349fb 100644 --- a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake +++ b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake @@ -1,7 +1,8 @@ include(CheckCXXCompilerFlag) include(CheckCCompilerFlag) +include(CheckLinkerFlag) -function(EnableCompilerFlag _flag _C _CXX) +function(EnableCompilerFlag _flag _C _CXX _LD) string(REGEX REPLACE "\\+" "PLUS" varname "${_flag}") string(REGEX REPLACE "[^A-Za-z0-9]+" "_" varname "${varname}") string(REGEX REPLACE "^_+" "" varname "${varname}") @@ -18,6 +19,13 @@ function(EnableCompilerFlag _flag _C _CXX) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_flag}" PARENT_SCOPE) endif () endif () + if (_LD) + CHECK_LINKER_FLAG(C ${_flag} LD_FLAG_${varname}) + if (LD_FLAG_${varname}) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${_flag}" PARENT_SCOPE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${_flag}" PARENT_SCOPE) + endif () + endif () endfunction() macro(ADD_ZSTD_COMPILATION_FLAGS) @@ -30,33 +38,39 @@ macro(ADD_ZSTD_COMPILATION_FLAGS) # EnableCompilerFlag("-std=c99" true false) # Set C compiation to c99 standard if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC) # clang-cl normally maps -Wall to -Weverything. - EnableCompilerFlag("/clang:-Wall" true true) + EnableCompilerFlag("/clang:-Wall" true true false) else () - EnableCompilerFlag("-Wall" true true) + EnableCompilerFlag("-Wall" true true false) endif () - EnableCompilerFlag("-Wextra" true true) - EnableCompilerFlag("-Wundef" true true) - EnableCompilerFlag("-Wshadow" true true) - EnableCompilerFlag("-Wcast-align" true true) - EnableCompilerFlag("-Wcast-qual" true true) - EnableCompilerFlag("-Wstrict-prototypes" true false) + EnableCompilerFlag("-Wextra" true true false) + EnableCompilerFlag("-Wundef" true true false) + EnableCompilerFlag("-Wshadow" true true false) + EnableCompilerFlag("-Wcast-align" true true false) + EnableCompilerFlag("-Wcast-qual" true true false) + EnableCompilerFlag("-Wstrict-prototypes" true false false) # Enable asserts in Debug mode if (CMAKE_BUILD_TYPE MATCHES "Debug") - EnableCompilerFlag("-DDEBUGLEVEL=1" true true) + EnableCompilerFlag("-DDEBUGLEVEL=1" true true false) endif () + # Add noexecstack flags + # LDFLAGS + EnableCompilerFlag("-z noexecstack" false false true) + # CFLAGS & CXXFLAGS + EnableCompilerFlag("-Qunused-arguments" true true false) + EnableCompilerFlag("-Wa,--noexecstack" true true false) elseif (MSVC) # Add specific compilation flags for Windows Visual set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)") if (CMAKE_GENERATOR MATCHES "Visual Studio" AND ACTIVATE_MULTITHREADED_COMPILATION) - EnableCompilerFlag("/MP" true true) + EnableCompilerFlag("/MP" true true false) endif () # UNICODE SUPPORT - EnableCompilerFlag("/D_UNICODE" true true) - EnableCompilerFlag("/DUNICODE" true true) + EnableCompilerFlag("/D_UNICODE" true true false) + EnableCompilerFlag("/DUNICODE" true true false) # Enable asserts in Debug mode if (CMAKE_BUILD_TYPE MATCHES "Debug") - EnableCompilerFlag("/DDEBUGLEVEL=1" true true) + EnableCompilerFlag("/DDEBUGLEVEL=1" true true false) endif () endif () diff --git a/build/cmake/contrib/CMakeLists.txt b/build/cmake/contrib/CMakeLists.txt index f7631d08c..8df2a17b3 100644 --- a/build/cmake/contrib/CMakeLists.txt +++ b/build/cmake/contrib/CMakeLists.txt @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/build/cmake/contrib/gen_html/CMakeLists.txt b/build/cmake/contrib/gen_html/CMakeLists.txt index 8fdd61131..d1ff6c64b 100644 --- a/build/cmake/contrib/gen_html/CMakeLists.txt +++ b/build/cmake/contrib/gen_html/CMakeLists.txt @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2015-present, Yann Collet, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt index 27af86c88..f7098fa0f 100644 --- a/build/cmake/contrib/pzstd/CMakeLists.txt +++ b/build/cmake/contrib/pzstd/CMakeLists.txt @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2016-present, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index cf7af0f8c..30349586b 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2015-present, Yann Collet, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the @@ -77,6 +77,12 @@ if (MSVC) set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc) endif () +# Explicitly set the language to C for all files, including ASM files. +# Our assembly expects to be compiled by a C compiler, and is only enabled for +# __GNUC__ compatible compilers. Otherwise all the ASM code is disabled by +# macros. +set_source_files_properties(${Sources} PROPERTIES LANGUAGE C) + # Split project to static and shared libraries build set(library_targets) if (ZSTD_BUILD_SHARED) diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt index 28b1e1d16..58d998e42 100644 --- a/build/cmake/programs/CMakeLists.txt +++ b/build/cmake/programs/CMakeLists.txt @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2015-present, Yann Collet, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt index 6c1ea298a..250f0508f 100644 --- a/build/cmake/tests/CMakeLists.txt +++ b/build/cmake/tests/CMakeLists.txt @@ -1,6 +1,6 @@ # ################################################################ # zstd - Makefile -# Copyright (C) Yann Collet 2014-present +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # BSD license @@ -27,7 +27,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # You can contact the author at : -# - zstd homepage : http://www.zstd.net/ +# - zstd homepage : https://facebook.github.io/zstd/ # ################################################################ project(tests) @@ -81,7 +81,7 @@ add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS}) # # zstreamtest # -add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c) +add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c ${TESTS_DIR}/external_matchfinder.c) if (NOT MSVC) target_compile_options(zstreamtest PRIVATE "-Wno-deprecated-declarations") endif() @@ -96,13 +96,14 @@ add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS}) AddTestFlagsOption(ZSTD_PLAYTESTS_FLAGS "$ENV{PLAYTESTS_FLAGS}" "Semicolon-separated list of flags to pass to the playTests.sh test") add_test(NAME playTests COMMAND sh -c "\"${TESTS_DIR}/playTests.sh\" ${ZSTD_PLAYTESTS_FLAGS}") -if (ZSTD_BUILD_PROGRAMS) +find_program(UNAME uname) # Run script only in unix shell environments +if (ZSTD_BUILD_PROGRAMS AND UNAME) set_property(TEST playTests APPEND PROPERTY ENVIRONMENT "ZSTD_BIN=$" "DATAGEN_BIN=$" ) else() - message(STATUS "Disabling playTests.sh test because ZSTD_BUILD_PROGRAMS is not enabled") + message(STATUS "Disabling playTests.sh test because requirements not met") set_tests_properties(playTests PROPERTIES DISABLED YES) endif() diff --git a/build/meson/contrib/pzstd/meson.build b/build/meson/contrib/pzstd/meson.build index 2c47999fa..b95dc7964 100644 --- a/build/meson/contrib/pzstd/meson.build +++ b/build/meson/contrib/pzstd/meson.build @@ -18,7 +18,8 @@ pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'), join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')] pzstd = executable('pzstd', pzstd_sources, - cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-Wno-deprecated-declarations' ], + cpp_args: pzstd_warning_flags, include_directories: pzstd_includes, dependencies: [ libzstd_dep, thread_dep ], + override_options: ['b_ndebug=true'], install: true) diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index 467007491..68db2ca6c 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -160,7 +160,7 @@ pkgconfig.generate(libzstd, filebase: 'libzstd', description: 'fast lossless compression algorithm library', version: zstd_libversion, - url: 'http://www.zstd.net/') + url: 'https://facebook.github.io/zstd/') install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'), join_paths(zstd_rootdir, 'lib/zdict.h'), diff --git a/build/meson/meson.build b/build/meson/meson.build index 98d556aa3..576dc44db 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -26,7 +26,7 @@ project('zstd', version: run_command( find_program('GetZstdLibraryVersion.py'), '../../lib/zstd.h', check: true).stdout().strip(), - meson_version: '>=0.48.0') + meson_version: '>=0.50.0') cc = meson.get_compiler('c') cxx = meson.get_compiler('cpp') @@ -104,8 +104,10 @@ use_lz4 = lz4_dep.found() add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c']) +pzstd_warning_flags = [] if [compiler_gcc, compiler_clang].contains(cc_id) common_warning_flags = [ '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ] + pzstd_warning_flags = ['-Wno-shadow', '-Wno-deprecated-declarations'] if cc_id == compiler_clang common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation'] endif diff --git a/build/meson/meson_options.txt b/build/meson/meson_options.txt index accf3fa10..f35cd5fc8 100644 --- a/build/meson/meson_options.txt +++ b/build/meson/meson_options.txt @@ -14,7 +14,7 @@ option('legacy_level', type: 'integer', min: 0, max: 7, value: 5, description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+') option('debug_level', type: 'integer', min: 0, max: 9, value: 1, description: 'Enable run-time debug. See lib/common/debug.h') -option('backtrace', type: 'boolean', value: false, +option('backtrace', type: 'feature', value: 'disabled', description: 'Display a stack backtrace when execution generates a runtime exception') option('static_runtime', type: 'boolean', value: false, description: 'Link to static run-time libraries on MSVC') diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build index 8df3a5dcb..e611dc337 100644 --- a/build/meson/programs/meson.build +++ b/build/meson/programs/meson.build @@ -51,7 +51,8 @@ endif export_dynamic_on_windows = false # explicit backtrace enable/disable for Linux & Darwin -if not use_backtrace +have_execinfo = cc.has_header('execinfo.h', required: use_backtrace) +if not have_execinfo zstd_c_args += '-DBACKTRACE_ENABLE=0' elif use_debug and host_machine_os == os_windows # MinGW target zstd_c_args += '-DBACKTRACE_ENABLE=1' diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build index 22f43209a..e70b73432 100644 --- a/build/meson/tests/meson.build +++ b/build/meson/tests/meson.build @@ -21,7 +21,6 @@ FUZZER_FLAGS = ['--no-big-tests'] FUZZERTEST = '-T200s' ZSTREAM_TESTTIME = '-T90s' DECODECORPUS_TESTTIME = '-T30' -ZSTDRTTEST = ['--test-large-data'] # ============================================================================= # Executables @@ -66,8 +65,10 @@ fuzzer = executable('fuzzer', dependencies: [ testcommon_dep, thread_dep ], install: false) -zstreamtest_sources = [join_paths(zstd_rootdir, 'tests/seqgen.c'), - join_paths(zstd_rootdir, 'tests/zstreamtest.c')] +zstreamtest_sources = [ + join_paths(zstd_rootdir, 'tests/seqgen.c'), + join_paths(zstd_rootdir, 'tests/zstreamtest.c'), + join_paths(zstd_rootdir, 'tests/external_matchfinder.c')] zstreamtest = executable('zstreamtest', zstreamtest_sources, include_directories: test_includes, @@ -134,24 +135,38 @@ checkTag = executable('checkTag', # ============================================================================= if tests_supported_oses.contains(host_machine_os) - valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: true) + valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: false) valgrindTest_py = files('valgrindTest.py') - test('valgrindTest', - valgrindTest_py, - args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench], - depends: [zstd, datagen, fuzzer, fullbench], - timeout: 600) # Timeout should work on HDD drive + if valgrind_prog.found() + test('valgrindTest', + valgrindTest_py, + args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench], + depends: [zstd, datagen, fuzzer, fullbench], + timeout: 600) # Timeout should work on HDD drive + endif endif if host_machine_os != os_windows playTests_sh = find_program(join_paths(zstd_rootdir, 'tests/playTests.sh'), required: true) - test('test-zstd', - playTests_sh, - args: ZSTDRTTEST, - env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'], - depends: [datagen], - workdir: meson.current_build_dir(), - timeout: 2800) # Timeout should work on HDD drive + + # add slow tests only if the meson version is new enough to support + # test setups with default-excluded suites + if meson.version().version_compare('>=0.57.0') + matrix = {'fast': [], 'slow': ['--test-large-data']} + else + matrix = {'fast': []} + endif + + foreach suite, opt: matrix + test('test-zstd-'+suite, + playTests_sh, + args: opt, + env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'], + depends: [datagen], + suite: suite, + workdir: meson.current_build_dir(), + timeout: 2800) # Timeout should work on HDD drive + endforeach endif test('test-fullbench-1', @@ -190,3 +205,11 @@ test('test-decodecorpus', args: ['-t', DECODECORPUS_TESTTIME], timeout: 60) test('test-poolTests', poolTests) # should be fast + +if meson.version().version_compare('>=0.57.0') + add_test_setup('fast', + is_default: true, + exclude_suites: ['slow']) + add_test_setup('slow', + exclude_suites: ['fast']) +endif diff --git a/build/single_file_libs/zstd-in.c b/build/single_file_libs/zstd-in.c index 59c6b5f19..e6fca9e4a 100644 --- a/build/single_file_libs/zstd-in.c +++ b/build/single_file_libs/zstd-in.c @@ -8,7 +8,7 @@ * \endcode */ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/build/single_file_libs/zstddeclib-in.c b/build/single_file_libs/zstddeclib-in.c index 5a58589c9..8d9c1f54b 100644 --- a/build/single_file_libs/zstddeclib-in.c +++ b/build/single_file_libs/zstddeclib-in.c @@ -8,7 +8,7 @@ * \endcode */ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/diagnose_corruption/Makefile b/contrib/diagnose_corruption/Makefile index a21a00212..ecc9e6395 100644 --- a/contrib/diagnose_corruption/Makefile +++ b/contrib/diagnose_corruption/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2019-present, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/diagnose_corruption/check_flipped_bits.c b/contrib/diagnose_corruption/check_flipped_bits.c index cc40ab84b..09ddd4674 100644 --- a/contrib/diagnose_corruption/check_flipped_bits.c +++ b/contrib/diagnose_corruption/check_flipped_bits.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/externalMatchfinder/.gitignore b/contrib/externalMatchfinder/.gitignore new file mode 100644 index 000000000..46357ef58 --- /dev/null +++ b/contrib/externalMatchfinder/.gitignore @@ -0,0 +1,2 @@ +# build artifacts +externalMatchfinder diff --git a/contrib/externalMatchfinder/Makefile b/contrib/externalMatchfinder/Makefile new file mode 100644 index 000000000..2baa558cb --- /dev/null +++ b/contrib/externalMatchfinder/Makefile @@ -0,0 +1,40 @@ +# ################################################################ +# Copyright (c) Yann Collet, Meta Platforms, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + +PROGDIR = ../../programs +LIBDIR = ../../lib + +LIBZSTD = $(LIBDIR)/libzstd.a + +CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/compress -I$(LIBDIR)/common + +CFLAGS ?= -O3 +CFLAGS += -std=gnu99 +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum \ + -Wstrict-prototypes -Wundef -Wpointer-arith \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) + +default: externalMatchfinder + +all: externalMatchfinder + +externalMatchfinder: matchfinder.c main.c $(LIBZSTD) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +.PHONY: $(LIBZSTD) +$(LIBZSTD): + $(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)" + +clean: + $(RM) *.o + $(MAKE) -C $(LIBDIR) clean > /dev/null + $(RM) externalMatchfinder diff --git a/contrib/externalMatchfinder/README.md b/contrib/externalMatchfinder/README.md new file mode 100644 index 000000000..cb7d49d97 --- /dev/null +++ b/contrib/externalMatchfinder/README.md @@ -0,0 +1,14 @@ +externalMatchfinder +===================== + +`externalMatchfinder` is a test tool for the external matchfinder API. +It demonstrates how to use the API to perform a simple round-trip test. + +A sample matchfinder is provided in matchfinder.c, but the user can swap +this out with a different one if desired. The sample matchfinder implements +LZ compression with a 1KB hashtable. Dictionary compression is not currently supported. + +Command line : +``` +externalMatchfinder filename +``` diff --git a/contrib/externalMatchfinder/main.c b/contrib/externalMatchfinder/main.c new file mode 100644 index 000000000..6971a46c7 --- /dev/null +++ b/contrib/externalMatchfinder/main.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) Yann Collet, Meta Platforms, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include +#include +#include +#include + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#include "zstd_errors.h" +#include "matchfinder.h" // simpleExternalMatchFinder + +#define CHECK(res) \ +do { \ + if (ZSTD_isError(res)) { \ + printf("ERROR: %s\n", ZSTD_getErrorName(res)); \ + return 1; \ + } \ +} while (0) \ + +int main(int argc, char *argv[]) { + if (argc != 2) { + printf("Usage: exampleMatchfinder \n"); + return 1; + } + + ZSTD_CCtx* const zc = ZSTD_createCCtx(); + + int simpleExternalMatchState = 0xdeadbeef; + + // Here is the crucial bit of code! + ZSTD_registerExternalMatchFinder( + zc, + &simpleExternalMatchState, + simpleExternalMatchFinder + ); + + { + size_t const res = ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 1); + CHECK(res); + } + + FILE *f = fopen(argv[1], "rb"); + assert(f); + { + int const ret = fseek(f, 0, SEEK_END); + assert(ret == 0); + } + size_t const srcSize = ftell(f); + { + int const ret = fseek(f, 0, SEEK_SET); + assert(ret == 0); + } + + char* const src = malloc(srcSize + 1); + assert(src); + { + size_t const ret = fread(src, srcSize, 1, f); + assert(ret == 1); + int const ret2 = fclose(f); + assert(ret2 == 0); + } + + size_t const dstSize = ZSTD_compressBound(srcSize); + char* const dst = malloc(dstSize); + assert(dst); + + size_t const cSize = ZSTD_compress2(zc, dst, dstSize, src, srcSize); + CHECK(cSize); + + char* const val = malloc(srcSize); + assert(val); + + { + size_t const res = ZSTD_decompress(val, srcSize, dst, cSize); + CHECK(res); + } + + if (memcmp(src, val, srcSize) == 0) { + printf("Compression and decompression were successful!\n"); + printf("Original size: %lu\n", srcSize); + printf("Compressed size: %lu\n", cSize); + } else { + printf("ERROR: input and validation buffers don't match!\n"); + for (size_t i = 0; i < srcSize; i++) { + if (src[i] != val[i]) { + printf("First bad index: %zu\n", i); + break; + } + } + return 1; + } + + ZSTD_freeCCtx(zc); + free(src); + free(dst); + free(val); + return 0; +} diff --git a/contrib/externalMatchfinder/matchfinder.c b/contrib/externalMatchfinder/matchfinder.c new file mode 100644 index 000000000..f119193ef --- /dev/null +++ b/contrib/externalMatchfinder/matchfinder.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) Yann Collet, Meta Platforms, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "matchfinder.h" + +#define HSIZE 1024 +static U32 const HLOG = 10; +static U32 const MLS = 4; +static U32 const BADIDX = 0xffffffff; + +size_t simpleExternalMatchFinder( + void* externalMatchState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +) { + const BYTE* const istart = (const BYTE*)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + const BYTE* anchor = istart; + size_t seqCount = 0; + U32 hashTable[HSIZE]; + + (void)externalMatchState; + (void)dict; + (void)dictSize; + (void)outSeqsCapacity; + (void)compressionLevel; + + { int i; + for (i=0; i < HSIZE; i++) { + hashTable[i] = BADIDX; + } } + + while (ip + MLS < iend) { + size_t const hash = ZSTD_hashPtr(ip, HLOG, MLS); + U32 const matchIndex = hashTable[hash]; + hashTable[hash] = (U32)(ip - istart); + + if (matchIndex != BADIDX) { + const BYTE* const match = istart + matchIndex; + U32 const matchLen = (U32)ZSTD_count(ip, match, iend); + if (matchLen >= ZSTD_MINMATCH_MIN) { + U32 const litLen = (U32)(ip - anchor); + U32 const offset = (U32)(ip - match); + ZSTD_Sequence const seq = { + offset, litLen, matchLen, 0 + }; + + /* Note: it's crucial to stay within the window size! */ + if (offset <= windowSize) { + outSeqs[seqCount++] = seq; + ip += matchLen; + anchor = ip; + continue; + } + } + } + + ip++; + } + + { ZSTD_Sequence const finalSeq = { + 0, (U32)(iend - anchor), 0, 0 + }; + outSeqs[seqCount++] = finalSeq; + } + + return seqCount; +} diff --git a/contrib/externalMatchfinder/matchfinder.h b/contrib/externalMatchfinder/matchfinder.h new file mode 100644 index 000000000..f8ba1c965 --- /dev/null +++ b/contrib/externalMatchfinder/matchfinder.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) Yann Collet, Meta Platforms, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MATCHFINDER_H +#define MATCHFINDER_H + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + +size_t simpleExternalMatchFinder( + void* externalMatchState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +#endif diff --git a/contrib/freestanding_lib/freestanding.py b/contrib/freestanding_lib/freestanding.py index 4e0a944f1..df6983245 100755 --- a/contrib/freestanding_lib/freestanding.py +++ b/contrib/freestanding_lib/freestanding.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # ################################################################ -# Copyright (c) 2021-2021, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/gen_html/Makefile b/contrib/gen_html/Makefile index 425f266c4..26e870399 100644 --- a/contrib/gen_html/Makefile +++ b/contrib/gen_html/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2016-present, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/gen_html/gen_html.cpp b/contrib/gen_html/gen_html.cpp index 90d5b21a3..1da5879ad 100644 --- a/contrib/gen_html/gen_html.cpp +++ b/contrib/gen_html/gen_html.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Przemyslaw Skibinski, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index 4c055b0ed..40734e62e 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2018-present, Yann Collet, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index cc489000f..eeaaf7182 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-present, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -856,7 +856,7 @@ int bench(const char **fileNameTable, unsigned nbFiles, const char *dictionary, CONTROL(cTotalSizeNoDict != 0); DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n", clevel, - (double)totalSrcSlicesSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict); + (double)totalSrcSlicesSize / (double)cTotalSizeNoDict, (unsigned)cTotalSizeNoDict); size_t* const cSizes = malloc(nbBlocks * sizeof(size_t)); CONTROL(cSizes != NULL); @@ -865,7 +865,7 @@ int bench(const char **fileNameTable, unsigned nbFiles, const char *dictionary, CONTROL(cTotalSize != 0); DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n", (unsigned)dictBuffer.size, - (double)totalSrcSlicesSize / cTotalSize, (unsigned)cTotalSize); + (double)totalSrcSlicesSize / (double)cTotalSize, (unsigned)cTotalSize); /* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */ shrinkSizes(dstSlices, cSizes); @@ -985,7 +985,7 @@ int usage(const char* exeName) DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT); DISPLAY ("-D # : use # as a dictionary (default: create one) \n"); DISPLAY ("-i# : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S); - DISPLAY ("-p# : print speed for all rounds 0=fastest 1=median (default: 0)"); + DISPLAY ("-p# : print speed for all rounds 0=fastest 1=median (default: 0) \n"); DISPLAY ("--nbBlocks=#: use # blocks for bench (default: one per file) \n"); DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n"); DISPLAY ("-h : help (this text) \n"); diff --git a/contrib/linux-kernel/Makefile b/contrib/linux-kernel/Makefile index baa1f24c6..63dd15d95 100644 --- a/contrib/linux-kernel/Makefile +++ b/contrib/linux-kernel/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the @@ -32,9 +32,7 @@ libzstd: -DZSTD_LEGACY_SUPPORT=0 \ -DZSTD_STATIC_LINKING_ONLY \ -DFSE_STATIC_LINKING_ONLY \ - -DHUF_STATIC_LINKING_ONLY \ -DXXH_STATIC_LINKING_ONLY \ - -DMEM_FORCE_MEMORY_ACCESS=0 \ -D__GNUC__ \ -D__linux__=1 \ -DSTATIC_BMI2=0 \ @@ -50,8 +48,8 @@ libzstd: -UZSTD_MULTITHREAD \ -U_MSC_VER \ -U_WIN32 \ - -RZSTDLIB_VISIBILITY= \ - -RZSTDERRORLIB_VISIBILITY= \ + -RZSTDLIB_VISIBLE= \ + -RZSTDERRORLIB_VISIBLE= \ -RZSTD_FALLTHROUGH=fallthrough \ -DZSTD_HAVE_WEAK_SYMBOLS=0 \ -DZSTD_TRACE=0 \ diff --git a/contrib/linux-kernel/decompress_sources.h b/contrib/linux-kernel/decompress_sources.h index a06ca187a..8a47eb2a4 100644 --- a/contrib/linux-kernel/decompress_sources.h +++ b/contrib/linux-kernel/decompress_sources.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */ /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/linux.mk b/contrib/linux-kernel/linux.mk index 20f08c644..464c410b2 100644 --- a/contrib/linux-kernel/linux.mk +++ b/contrib/linux-kernel/linux.mk @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause # ################################################################ -# Copyright (c) Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/linux_zstd.h b/contrib/linux-kernel/linux_zstd.h index 113408eef..f109d49f4 100644 --- a/contrib/linux-kernel/linux_zstd.h +++ b/contrib/linux-kernel/linux_zstd.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/mem.h b/contrib/linux-kernel/mem.h index 1d9cc0392..a7231822b 100644 --- a/contrib/linux-kernel/mem.h +++ b/contrib/linux-kernel/mem.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/Makefile b/contrib/linux-kernel/test/Makefile index 53b0c2a65..67b55e665 100644 --- a/contrib/linux-kernel/test/Makefile +++ b/contrib/linux-kernel/test/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/include/linux/compiler.h b/contrib/linux-kernel/test/include/linux/compiler.h index de43edb69..988ce4a20 100644 --- a/contrib/linux-kernel/test/include/linux/compiler.h +++ b/contrib/linux-kernel/test/include/linux/compiler.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/include/linux/errno.h b/contrib/linux-kernel/test/include/linux/errno.h index b24752257..b4bdcba0e 100644 --- a/contrib/linux-kernel/test/include/linux/errno.h +++ b/contrib/linux-kernel/test/include/linux/errno.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/include/linux/kernel.h b/contrib/linux-kernel/test/include/linux/kernel.h index 1f702abac..a4d791cd1 100644 --- a/contrib/linux-kernel/test/include/linux/kernel.h +++ b/contrib/linux-kernel/test/include/linux/kernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/include/linux/limits.h b/contrib/linux-kernel/test/include/linux/limits.h index db9c09904..574aa7b34 100644 --- a/contrib/linux-kernel/test/include/linux/limits.h +++ b/contrib/linux-kernel/test/include/linux/limits.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/include/linux/math64.h b/contrib/linux-kernel/test/include/linux/math64.h index 8eefa2d5c..7f6713e73 100644 --- a/contrib/linux-kernel/test/include/linux/math64.h +++ b/contrib/linux-kernel/test/include/linux/math64.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/include/linux/module.h b/contrib/linux-kernel/test/include/linux/module.h index 63a28d57b..06ef56f9e 100644 --- a/contrib/linux-kernel/test/include/linux/module.h +++ b/contrib/linux-kernel/test/include/linux/module.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/include/linux/printk.h b/contrib/linux-kernel/test/include/linux/printk.h index eab08e0c4..92a25278e 100644 --- a/contrib/linux-kernel/test/include/linux/printk.h +++ b/contrib/linux-kernel/test/include/linux/printk.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/include/linux/stddef.h b/contrib/linux-kernel/test/include/linux/stddef.h index 8538eb3e4..15c7408fc 100644 --- a/contrib/linux-kernel/test/include/linux/stddef.h +++ b/contrib/linux-kernel/test/include/linux/stddef.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/include/linux/swab.h b/contrib/linux-kernel/test/include/linux/swab.h index 783046b42..2b48b434c 100644 --- a/contrib/linux-kernel/test/include/linux/swab.h +++ b/contrib/linux-kernel/test/include/linux/swab.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/include/linux/types.h b/contrib/linux-kernel/test/include/linux/types.h index 459a45700..b413db6f9 100644 --- a/contrib/linux-kernel/test/include/linux/types.h +++ b/contrib/linux-kernel/test/include/linux/types.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/include/linux/xxhash.h b/contrib/linux-kernel/test/include/linux/xxhash.h index 7e92a706e..d41cbd933 100644 --- a/contrib/linux-kernel/test/include/linux/xxhash.h +++ b/contrib/linux-kernel/test/include/linux/xxhash.h @@ -2,7 +2,7 @@ * xxHash - Extremely Fast Hash algorithm * Copyright (C) 2012-2016, Yann Collet. * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + * BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are @@ -260,7 +260,7 @@ XXH_API void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state * xxHash - Extremely Fast Hash algorithm * Copyright (C) 2012-2016, Yann Collet. * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + * BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are diff --git a/contrib/linux-kernel/test/static_test.c b/contrib/linux-kernel/test/static_test.c index d2b8b5a32..ba4a420d4 100644 --- a/contrib/linux-kernel/test/static_test.c +++ b/contrib/linux-kernel/test/static_test.c @@ -1,5 +1,5 @@ /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/test/test.c b/contrib/linux-kernel/test/test.c index 6cd1730bb..0f4ba3f45 100644 --- a/contrib/linux-kernel/test/test.c +++ b/contrib/linux-kernel/test/test.c @@ -1,5 +1,5 @@ /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -186,11 +186,14 @@ static void __attribute__((noinline)) use(void *x) { asm volatile("" : "+r"(x)); } +static void __attribute__((noinline)) fill_stack(void) { + memset(g_stack, 0x33, 8192); +} + static void __attribute__((noinline)) set_stack(void) { char stack[8192]; g_stack = stack; - memset(g_stack, 0x33, 8192); use(g_stack); } @@ -208,6 +211,7 @@ static void __attribute__((noinline)) check_stack(void) { static void test_stack_usage(test_data_t const *data) { set_stack(); + fill_stack(); test_f2fs(); test_btrfs(data); test_decompress_unzstd(data); diff --git a/contrib/linux-kernel/zstd_common_module.c b/contrib/linux-kernel/zstd_common_module.c index 22686e367..2fead39eb 100644 --- a/contrib/linux-kernel/zstd_common_module.c +++ b/contrib/linux-kernel/zstd_common_module.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/zstd_compress_module.c b/contrib/linux-kernel/zstd_compress_module.c index 04e1b5c01..8ecf43226 100644 --- a/contrib/linux-kernel/zstd_compress_module.c +++ b/contrib/linux-kernel/zstd_compress_module.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/zstd_decompress_module.c b/contrib/linux-kernel/zstd_decompress_module.c index f4ed952ed..eb1c49e69 100644 --- a/contrib/linux-kernel/zstd_decompress_module.c +++ b/contrib/linux-kernel/zstd_decompress_module.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/linux-kernel/zstd_deps.h b/contrib/linux-kernel/zstd_deps.h index 7a5bf4483..925161416 100644 --- a/contrib/linux-kernel/zstd_deps.h +++ b/contrib/linux-kernel/zstd_deps.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */ /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/match_finders/zstd_edist.c b/contrib/match_finders/zstd_edist.c index d63a7cf8d..d685cdd9e 100644 --- a/contrib/match_finders/zstd_edist.c +++ b/contrib/match_finders/zstd_edist.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/match_finders/zstd_edist.h b/contrib/match_finders/zstd_edist.h index a947649b2..c739e2abc 100644 --- a/contrib/match_finders/zstd_edist.h +++ b/contrib/match_finders/zstd_edist.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/ErrorHolder.h b/contrib/pzstd/ErrorHolder.h index 829651c59..2c2797ede 100644 --- a/contrib/pzstd/ErrorHolder.h +++ b/contrib/pzstd/ErrorHolder.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/Logging.h b/contrib/pzstd/Logging.h index beb160b64..84a08d200 100644 --- a/contrib/pzstd/Logging.h +++ b/contrib/pzstd/Logging.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index 3d930cca9..830053cd7 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2016-present, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp index 90f9d571f..a77824edb 100644 --- a/contrib/pzstd/Options.cpp +++ b/contrib/pzstd/Options.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/Options.h b/contrib/pzstd/Options.h index 924543abf..92c18a350 100644 --- a/contrib/pzstd/Options.h +++ b/contrib/pzstd/Options.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index 2c09bda7a..67b941991 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -10,11 +10,13 @@ #include "Pzstd.h" #include "SkippableFrame.h" #include "utils/FileSystem.h" +#include "utils/Portability.h" #include "utils/Range.h" #include "utils/ScopeGuard.h" #include "utils/ThreadPool.h" #include "utils/WorkQueue.h" +#include #include #include #include @@ -336,6 +338,10 @@ static size_t calculateStep( const ZSTD_parameters ¶ms) { (void)size; (void)numThreads; + // Not validated to work correctly for window logs > 23. + // It will definitely fail if windowLog + 2 is >= 4GB because + // the skippable frame can only store sizes up to 4GB. + assert(params.cParams.windowLog <= 23); return size_t{1} << (params.cParams.windowLog + 2); } @@ -587,7 +593,8 @@ std::uint64_t writeFile( // start writing before compression is done because we need to know the // compressed size. // Wait for the compressed size to be available and write skippable frame - SkippableFrame frame(out->size()); + assert(uint64_t(out->size()) < uint64_t(1) << 32); + SkippableFrame frame(uint32_t(out->size())); if (!writeData(frame.data(), outputFd)) { errorHolder.setError("Failed to write output"); return bytesWritten; diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h index c667c887d..3645e5942 100644 --- a/contrib/pzstd/Pzstd.h +++ b/contrib/pzstd/Pzstd.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/README.md b/contrib/pzstd/README.md index 84d945815..bc8f831f8 100644 --- a/contrib/pzstd/README.md +++ b/contrib/pzstd/README.md @@ -31,7 +31,7 @@ If this number is not suitable, during compilation you can define `PZSTD_NUM_THR ## Benchmarks -As a reference, PZstandard and Pigz were compared on an Intel Core i7 @ 3.1 GHz, each using 4 threads, with the [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia). +As a reference, PZstandard and Pigz were compared on an Intel Core i7 @ 3.1 GHz, each using 4 threads, with the [Silesia compression corpus](https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia). Compression Speed vs Ratio with 4 Threads | Decompression Speed with 4 Threads ------------------------------------------|----------------------------------- diff --git a/contrib/pzstd/SkippableFrame.cpp b/contrib/pzstd/SkippableFrame.cpp index 769866dfc..3bea4eb65 100644 --- a/contrib/pzstd/SkippableFrame.cpp +++ b/contrib/pzstd/SkippableFrame.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/SkippableFrame.h b/contrib/pzstd/SkippableFrame.h index 60deed040..817415e92 100644 --- a/contrib/pzstd/SkippableFrame.h +++ b/contrib/pzstd/SkippableFrame.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/main.cpp b/contrib/pzstd/main.cpp index b93f043b1..422b4a56a 100644 --- a/contrib/pzstd/main.cpp +++ b/contrib/pzstd/main.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp index e60114825..91e39750d 100644 --- a/contrib/pzstd/test/OptionsTest.cpp +++ b/contrib/pzstd/test/OptionsTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp index 5c7d66310..75453f5f6 100644 --- a/contrib/pzstd/test/PzstdTest.cpp +++ b/contrib/pzstd/test/PzstdTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/test/RoundTrip.h b/contrib/pzstd/test/RoundTrip.h index c6364ecb4..f777622a3 100644 --- a/contrib/pzstd/test/RoundTrip.h +++ b/contrib/pzstd/test/RoundTrip.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/test/RoundTripTest.cpp b/contrib/pzstd/test/RoundTripTest.cpp index 36af0673a..c37646d1c 100644 --- a/contrib/pzstd/test/RoundTripTest.cpp +++ b/contrib/pzstd/test/RoundTripTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/Buffer.h b/contrib/pzstd/utils/Buffer.h index d17ad2f2c..a85f770ba 100644 --- a/contrib/pzstd/utils/Buffer.h +++ b/contrib/pzstd/utils/Buffer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/FileSystem.h b/contrib/pzstd/utils/FileSystem.h index 3cfbe86e5..8d57d05f0 100644 --- a/contrib/pzstd/utils/FileSystem.h +++ b/contrib/pzstd/utils/FileSystem.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -8,16 +8,18 @@ */ #pragma once +#include "utils/Portability.h" #include "utils/Range.h" #include #include #include +#include #include // A small subset of `std::filesystem`. // `std::filesystem` should be a drop in replacement. -// See http://en.cppreference.com/w/cpp/filesystem for documentation. +// See https://en.cppreference.com/w/cpp/filesystem for documentation. namespace pzstd { @@ -28,7 +30,7 @@ typedef struct ::_stat64 file_status; typedef struct ::stat file_status; #endif -/// http://en.cppreference.com/w/cpp/filesystem/status +/// https://en.cppreference.com/w/cpp/filesystem/status inline file_status status(StringPiece path, std::error_code& ec) noexcept { file_status status; #if defined(_MSC_VER) @@ -44,7 +46,7 @@ inline file_status status(StringPiece path, std::error_code& ec) noexcept { return status; } -/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file +/// https://en.cppreference.com/w/cpp/filesystem/is_regular_file inline bool is_regular_file(file_status status) noexcept { #if defined(S_ISREG) return S_ISREG(status.st_mode); @@ -55,12 +57,12 @@ inline bool is_regular_file(file_status status) noexcept { #endif } -/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file +/// https://en.cppreference.com/w/cpp/filesystem/is_regular_file inline bool is_regular_file(StringPiece path, std::error_code& ec) noexcept { return is_regular_file(status(path, ec)); } -/// http://en.cppreference.com/w/cpp/filesystem/is_directory +/// https://en.cppreference.com/w/cpp/filesystem/is_directory inline bool is_directory(file_status status) noexcept { #if defined(S_ISDIR) return S_ISDIR(status.st_mode); @@ -71,22 +73,22 @@ inline bool is_directory(file_status status) noexcept { #endif } -/// http://en.cppreference.com/w/cpp/filesystem/is_directory +/// https://en.cppreference.com/w/cpp/filesystem/is_directory inline bool is_directory(StringPiece path, std::error_code& ec) noexcept { return is_directory(status(path, ec)); } -/// http://en.cppreference.com/w/cpp/filesystem/file_size +/// https://en.cppreference.com/w/cpp/filesystem/file_size inline std::uintmax_t file_size( StringPiece path, std::error_code& ec) noexcept { auto stat = status(path, ec); if (ec) { - return -1; + return std::numeric_limits::max(); } if (!is_regular_file(stat)) { ec.assign(ENOTSUP, std::generic_category()); - return -1; + return std::numeric_limits::max(); } ec.clear(); return stat.st_size; diff --git a/contrib/pzstd/utils/Likely.h b/contrib/pzstd/utils/Likely.h index 7cea8da27..52243a64e 100644 --- a/contrib/pzstd/utils/Likely.h +++ b/contrib/pzstd/utils/Likely.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/Portability.h b/contrib/pzstd/utils/Portability.h new file mode 100644 index 000000000..ef1f86e51 --- /dev/null +++ b/contrib/pzstd/utils/Portability.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +#pragma once + +#include + +// Required for windows, which defines min/max, but we want the std:: version. +#undef min +#undef max diff --git a/contrib/pzstd/utils/Range.h b/contrib/pzstd/utils/Range.h index 6a850ad4e..0fd8f9f86 100644 --- a/contrib/pzstd/utils/Range.h +++ b/contrib/pzstd/utils/Range.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -14,7 +14,9 @@ #pragma once #include "utils/Likely.h" +#include "utils/Portability.h" +#include #include #include #include diff --git a/contrib/pzstd/utils/ResourcePool.h b/contrib/pzstd/utils/ResourcePool.h index 8dfcdd765..7c4bb6235 100644 --- a/contrib/pzstd/utils/ResourcePool.h +++ b/contrib/pzstd/utils/ResourcePool.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/ScopeGuard.h b/contrib/pzstd/utils/ScopeGuard.h index c26f911bf..911fd9842 100644 --- a/contrib/pzstd/utils/ScopeGuard.h +++ b/contrib/pzstd/utils/ScopeGuard.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/ThreadPool.h b/contrib/pzstd/utils/ThreadPool.h index 8ece8e0da..a087d7c1c 100644 --- a/contrib/pzstd/utils/ThreadPool.h +++ b/contrib/pzstd/utils/ThreadPool.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/WorkQueue.h b/contrib/pzstd/utils/WorkQueue.h index 1d14d922c..d7947b814 100644 --- a/contrib/pzstd/utils/WorkQueue.h +++ b/contrib/pzstd/utils/WorkQueue.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/test/BufferTest.cpp b/contrib/pzstd/utils/test/BufferTest.cpp index fbba74e82..58bf08dcd 100644 --- a/contrib/pzstd/utils/test/BufferTest.cpp +++ b/contrib/pzstd/utils/test/BufferTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/test/RangeTest.cpp b/contrib/pzstd/utils/test/RangeTest.cpp index 755b50fa6..8b7dee271 100644 --- a/contrib/pzstd/utils/test/RangeTest.cpp +++ b/contrib/pzstd/utils/test/RangeTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/test/ResourcePoolTest.cpp b/contrib/pzstd/utils/test/ResourcePoolTest.cpp index 6fe145180..750ee084b 100644 --- a/contrib/pzstd/utils/test/ResourcePoolTest.cpp +++ b/contrib/pzstd/utils/test/ResourcePoolTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/test/ScopeGuardTest.cpp b/contrib/pzstd/utils/test/ScopeGuardTest.cpp index 7bc624da7..0f77cdf38 100644 --- a/contrib/pzstd/utils/test/ScopeGuardTest.cpp +++ b/contrib/pzstd/utils/test/ScopeGuardTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/test/ThreadPoolTest.cpp b/contrib/pzstd/utils/test/ThreadPoolTest.cpp index 703fd4c9c..a01052e60 100644 --- a/contrib/pzstd/utils/test/ThreadPoolTest.cpp +++ b/contrib/pzstd/utils/test/ThreadPoolTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/pzstd/utils/test/WorkQueueTest.cpp b/contrib/pzstd/utils/test/WorkQueueTest.cpp index 14cf77304..16600bb60 100644 --- a/contrib/pzstd/utils/test/WorkQueueTest.cpp +++ b/contrib/pzstd/utils/test/WorkQueueTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/recovery/Makefile b/contrib/recovery/Makefile index 9a9f4f2e8..be6ea4b0e 100644 --- a/contrib/recovery/Makefile +++ b/contrib/recovery/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2019-present, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/recovery/recover_directory.c b/contrib/recovery/recover_directory.c index 13f83fd10..b9bd7ab49 100644 --- a/contrib/recovery/recover_directory.c +++ b/contrib/recovery/recover_directory.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/seekable_format/examples/Makefile b/contrib/seekable_format/examples/Makefile index 9df6b75fb..fcc04587c 100644 --- a/contrib/seekable_format/examples/Makefile +++ b/contrib/seekable_format/examples/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2017-present, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/seekable_format/examples/parallel_compression.c b/contrib/seekable_format/examples/parallel_compression.c index 4118b0ad7..0ec9fbd20 100644 --- a/contrib/seekable_format/examples/parallel_compression.c +++ b/contrib/seekable_format/examples/parallel_compression.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/seekable_format/examples/parallel_processing.c b/contrib/seekable_format/examples/parallel_processing.c index 36226b49f..b1709db77 100644 --- a/contrib/seekable_format/examples/parallel_processing.c +++ b/contrib/seekable_format/examples/parallel_processing.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/seekable_format/examples/seekable_compression.c b/contrib/seekable_format/examples/seekable_compression.c index 9a331a895..182b46f64 100644 --- a/contrib/seekable_format/examples/seekable_compression.c +++ b/contrib/seekable_format/examples/seekable_compression.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/seekable_format/examples/seekable_decompression.c b/contrib/seekable_format/examples/seekable_decompression.c index e9e201333..2c4f3ba0c 100644 --- a/contrib/seekable_format/examples/seekable_decompression.c +++ b/contrib/seekable_format/examples/seekable_decompression.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/seekable_format/examples/seekable_decompression_mem.c b/contrib/seekable_format/examples/seekable_decompression_mem.c index e7b1c6505..44a06fbbf 100644 --- a/contrib/seekable_format/examples/seekable_decompression_mem.c +++ b/contrib/seekable_format/examples/seekable_decompression_mem.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/seekable_format/tests/Makefile b/contrib/seekable_format/tests/Makefile index d51deb3ea..a81f2229f 100644 --- a/contrib/seekable_format/tests/Makefile +++ b/contrib/seekable_format/tests/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2017-present, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/contrib/seekable_format/tests/seekable_tests.c b/contrib/seekable_format/tests/seekable_tests.c index a482638b9..1bb2d0e81 100644 --- a/contrib/seekable_format/tests/seekable_tests.c +++ b/contrib/seekable_format/tests/seekable_tests.c @@ -186,6 +186,40 @@ int main(int argc, const char** argv) } printf("Success!\n"); + + printf("Test %u - check ZSTD magic in compressing empty string: ", testNb++); + { // compressing empty string should return a zstd header + size_t const capacity = 255; + char* inBuffer = malloc(capacity); + assert(inBuffer != NULL); + inBuffer[0] = '\0'; + void* const outBuffer = malloc(capacity); + assert(outBuffer != NULL); + + ZSTD_seekable_CStream *s = ZSTD_seekable_createCStream(); + ZSTD_seekable_initCStream(s, 1, 1, 255); + + ZSTD_inBuffer input = { .src=inBuffer, .pos=0, .size=0 }; + ZSTD_outBuffer output = { .dst=outBuffer, .pos=0, .size=capacity }; + + ZSTD_seekable_compressStream(s, &output, &input); + ZSTD_seekable_endStream(s, &output); + + if((((char*)output.dst)[0] != '\x28') | (((char*)output.dst)[1] != '\xb5') | (((char*)output.dst)[2] != '\x2f') | (((char*)output.dst)[3] != '\xfd')) { + printf("%#02x %#02x %#02x %#02x\n", ((char*)output.dst)[0], ((char*)output.dst)[1] , ((char*)output.dst)[2] , ((char*)output.dst)[3] ); + + free(inBuffer); + free(outBuffer); + ZSTD_seekable_freeCStream(s); + goto _test_error; + } + + free(inBuffer); + free(outBuffer); + ZSTD_seekable_freeCStream(s); + } + printf("Success!\n"); + /* TODO: Add more tests */ printf("Finished tests\n"); return 0; diff --git a/contrib/seekable_format/zstd_seekable_compression_format.md b/contrib/seekable_format/zstd_seekable_compression_format.md index 55aebfd2e..7bd0790e8 100644 --- a/contrib/seekable_format/zstd_seekable_compression_format.md +++ b/contrib/seekable_format/zstd_seekable_compression_format.md @@ -2,7 +2,7 @@ ### Notices -Copyright (c) 2017-present Facebook, Inc. +Copyright (c) Meta Platforms, Inc. and affiliates. Permission is granted to copy and distribute this document for any purpose and without charge, diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c index 242bd2ac3..113f6f99c 100644 --- a/contrib/seekable_format/zstdseek_compress.c +++ b/contrib/seekable_format/zstdseek_compress.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -350,7 +350,7 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output) size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output) { - if (!zcs->writingSeekTable && zcs->frameDSize) { + if (!zcs->writingSeekTable) { const size_t endFrame = ZSTD_seekable_endFrame(zcs, output); if (ZSTD_isError(endFrame)) return endFrame; /* return an accurate size hint */ diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c index 0848d2519..fbb2d4fe3 100644 --- a/contrib/seekable_format/zstdseek_decompress.c +++ b/contrib/seekable_format/zstdseek_decompress.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-present, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -38,7 +38,7 @@ * note: it's better to use unistd.h's _POSIX_VERSION whenever possible */ # define PLATFORM_POSIX_VERSION 200112L -/* try to determine posix version through official unistd.h's _POSIX_VERSION (http://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html). +/* try to determine posix version through official unistd.h's _POSIX_VERSION (https://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html). * note : there is no simple way to know in advance if is present or not on target system, * Posix specification mandates its presence and its content, but target system must respect this spec. * It's necessary to _not_ #include whenever target OS is not unix-like @@ -517,9 +517,9 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign size_t forwardProgress; if (zs->decompressedOffset < offset) { /* dummy decompressions until we get to the target offset */ - outTmp = (ZSTD_outBuffer){zs->outBuff, MIN(SEEKABLE_BUFF_SIZE, offset - zs->decompressedOffset), 0}; + outTmp = (ZSTD_outBuffer){zs->outBuff, (size_t) (MIN(SEEKABLE_BUFF_SIZE, offset - zs->decompressedOffset)), 0}; } else { - outTmp = (ZSTD_outBuffer){dst, len, zs->decompressedOffset - offset}; + outTmp = (ZSTD_outBuffer){dst, len, (size_t) (zs->decompressedOffset - offset)}; } prevOutPos = outTmp.pos; diff --git a/contrib/seqBench/Makefile b/contrib/seqBench/Makefile index 0782961eb..e7f08a42c 100644 --- a/contrib/seqBench/Makefile +++ b/contrib/seqBench/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) 2018-present, Yann Collet, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/doc/README.md b/doc/README.md index bb7a3e490..8f3babcdb 100644 --- a/doc/README.md +++ b/doc/README.md @@ -5,8 +5,9 @@ This directory contains material defining the Zstandard format, as well as detailed instructions to use `zstd` library. __`zstd_manual.html`__ : Documentation of `zstd.h` API, in html format. -Click on this link: [http://zstd.net/zstd_manual.html](http://zstd.net/zstd_manual.html) -to display documentation of latest release in readable format within a browser. +Unfortunately, Github doesn't display `html` files in parsed format, just as source code. +For a readable display of html API documentation of latest release, +use this link: [https://raw.githack.com/facebook/zstd/release/doc/zstd_manual.html](https://raw.githack.com/facebook/zstd/release/doc/zstd_manual.html) . __`zstd_compression_format.md`__ : This document defines the Zstandard compression format. Compliant decoders must adhere to this document, diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile index a9c601ebc..f6deeb13d 100644 --- a/doc/educational_decoder/Makefile +++ b/doc/educational_decoder/Makefile @@ -1,5 +1,5 @@ # ################################################################ -# Copyright (c) Yann Collet, Facebook, Inc. +# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c index 935f60da8..12c5a801b 100644 --- a/doc/educational_decoder/harness.c +++ b/doc/educational_decoder/harness.c @@ -1,5 +1,5 @@ /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index 936407086..9ade76502 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -1,5 +1,5 @@ /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -1890,7 +1890,7 @@ static size_t HUF_decompress_4stream(const HUF_dtable *const dtable, /// Initializes a Huffman table using canonical Huffman codes /// For more explanation on canonical Huffman codes see -/// http://www.cs.uofs.edu/~mccloske/courses/cmps340/huff_canonical_dec2015.html +/// https://www.cs.scranton.edu/~mccloske/courses/cmps340/huff_canonical_dec2015.html /// Codes within a level are allocated in symbol order (i.e. smaller symbols get /// earlier codes) static void HUF_init_dtable(HUF_dtable *const table, const u8 *const bits, diff --git a/doc/educational_decoder/zstd_decompress.h b/doc/educational_decoder/zstd_decompress.h index d89c83523..c13c8134d 100644 --- a/doc/educational_decoder/zstd_decompress.h +++ b/doc/educational_decoder/zstd_decompress.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md index 419de38eb..e40677a1d 100644 --- a/doc/zstd_compression_format.md +++ b/doc/zstd_compression_format.md @@ -3,7 +3,7 @@ Zstandard Compression Format ### Notices -Copyright (c) 2016-2021 Yann Collet, Facebook, Inc. +Copyright (c) Meta Platforms, Inc. and affiliates. Permission is granted to copy and distribute this document for any purpose and without charge, @@ -26,7 +26,7 @@ The purpose of this document is to define a lossless compressed data format, that is independent of CPU type, operating system, file system and character set, suitable for file compression, pipe and streaming compression, -using the [Zstandard algorithm](http://www.zstandard.org). +using the [Zstandard algorithm](https://facebook.github.io/zstd/). The text of the specification assumes a basic background in programming at the level of bits and other primitive data representations. @@ -35,7 +35,7 @@ even for an arbitrarily long sequentially presented input data stream, using only an a priori bounded amount of intermediate storage, and hence can be used in data communications. The format uses the Zstandard compression method, -and optional [xxHash-64 checksum method](http://www.xxhash.org), +and optional [xxHash-64 checksum method](https://cyan4973.github.io/xxHash/), for detection of data corruption. The data format defined by this specification @@ -134,7 +134,7 @@ __`Content_Checksum`__ An optional 32-bit checksum, only present if `Content_Checksum_flag` is set. The content checksum is the result -of [xxh64() hash function](http://www.xxhash.org) +of [xxh64() hash function](https://cyan4973.github.io/xxHash/) digesting the original (decoded) data as input, and a seed of zero. The low 4 bytes of the checksum are stored in __little-endian__ format. @@ -435,7 +435,7 @@ They can be decoded first, and then copied during [Sequence Execution], or they can be decoded on the flow during [Sequence Execution]. Literals can be stored uncompressed or compressed using Huffman prefix codes. -When compressed, an optional tree description can be present, +When compressed, a tree description may optionally be present, followed by 1 or 4 streams. | `Literals_Section_Header` | [`Huffman_Tree_Description`] | [jumpTable] | Stream1 | [Stream2] | [Stream3] | [Stream4] | @@ -510,7 +510,7 @@ Its value is : `Size_Format = (Literals_Section_Header[0]>>2) & 3` `Regenerated_Size = (Literals_Section_Header[0]>>4) + (Literals_Section_Header[1]<<4) + (Literals_Section_Header[2]<<12)` Only Stream1 is present for these cases. -Note : it's allowed to represent a short value (for example `13`) +Note : it's allowed to represent a short value (for example `27`) using a long format, even if it's less efficient. __`Size_Format` for `Compressed_Literals_Block` and `Treeless_Literals_Block`__ : @@ -521,19 +521,33 @@ __`Size_Format` for `Compressed_Literals_Block` and `Treeless_Literals_Block`__ Both `Regenerated_Size` and `Compressed_Size` use 10 bits (0-1023). `Literals_Section_Header` uses 3 bytes. - `Size_Format` == 01 : 4 streams. - Both `Regenerated_Size` and `Compressed_Size` use 10 bits (0-1023). + Both `Regenerated_Size` and `Compressed_Size` use 10 bits (6-1023). `Literals_Section_Header` uses 3 bytes. - `Size_Format` == 10 : 4 streams. - Both `Regenerated_Size` and `Compressed_Size` use 14 bits (0-16383). + Both `Regenerated_Size` and `Compressed_Size` use 14 bits (6-16383). `Literals_Section_Header` uses 4 bytes. - `Size_Format` == 11 : 4 streams. - Both `Regenerated_Size` and `Compressed_Size` use 18 bits (0-262143). + Both `Regenerated_Size` and `Compressed_Size` use 18 bits (6-262143). `Literals_Section_Header` uses 5 bytes. Both `Compressed_Size` and `Regenerated_Size` fields follow __little-endian__ convention. Note: `Compressed_Size` __includes__ the size of the Huffman Tree description _when_ it is present. +4 streams is superior to 1 stream in decompression speed, +by exploiting instruction level parallelism. +But it's also more expensive, +costing on average ~7.3 bytes more than the 1 stream mode, mostly from the jump table. + +In general, use the 4 streams mode when there are more literals to decode, +to favor higher decompression speeds. +Beyond 1KB, the 4 streams mode is compulsory anyway. + +Note that a minimum of 6 bytes is required for the 4 streams mode. +That's a technical minimum, but it's not recommended to employ the 4 streams mode +for such a small quantity, that would be wasteful. +A more practical lower bound would be around ~256 bytes. + #### Raw Literals Block The data in Stream1 is `Regenerated_Size` bytes long, it contains the raw literals data to be used during [Sequence Execution]. @@ -967,7 +981,7 @@ into a flow of concatenated frames. Skippable frames defined in this specification are compatible with [LZ4] ones. -[LZ4]:http://www.lz4.org +[LZ4]:https://lz4.github.io/lz4/ From a compliant decoder perspective, skippable frames need just be skipped, and their content ignored, resuming decoding after the skippable frame. diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index cca287b72..2ea5c2010 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1,10 +1,10 @@ -zstd 1.5.3 Manual +zstd 1.5.4 Manual -

zstd 1.5.3 Manual

+

zstd 1.5.4 Manual

Contents

    @@ -121,8 +121,8 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); note 6 : This function replaces ZSTD_getDecompressedSize()


    -
    ZSTDLIB_API
-ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+
    ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+ZSTDLIB_API
 unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
 
      NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
   Both functions work the same way, but ZSTD_getDecompressedSize() blends
@@ -139,8 +139,30 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
         or an error code if input is invalid 
 

    
 
-
    Helper functions
    #define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+
    Helper functions
    /* ZSTD_compressBound() :
+ * maximum compressed size in worst case single-pass scenario.
+ * When invoking `ZSTD_compress()` or any other one-pass compression function,
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ * as it eliminates one potential failure scenario,
+ * aka not enough room in dst buffer to write the compressed frame.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
+ *        In which case, ZSTD_compressBound() will return an error code
+ *        which can be tested using ZSTD_isError().
+ *
+ * ZSTD_COMPRESSBOUND() :
+ * same as ZSTD_compressBound(), but as a macro.
+ * It can be used to produce constants, which can be useful for static allocation,
+ * for example to size a static array on stack.
+ * Will produce constant value 0 if srcSize too large.
+ */
+#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
+#define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
+size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+/* ZSTD_isError() :
+ * Most ZSTD_* functions returning a size_t value can be tested for error,
+ * using ZSTD_isError().
+ * @return 1 if error, 0 otherwise
+ */
 unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
 const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
 int         ZSTD_minCLevel(void);               /*!< minimum negative compression level allowed, requires v1.4.0+ */
@@ -362,6 +384,8 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  /* accept NULL pointer */
      * ZSTD_c_useBlockSplitter
      * ZSTD_c_useRowMatchFinder
      * ZSTD_c_prefetchCDictTables
+     * ZSTD_c_enableMatchFinderFallback
+     * ZSTD_c_maxBlockSize
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -381,7 +405,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  /* accept NULL pointer */
      ZSTD_c_experimentalParam13=1010,
      ZSTD_c_experimentalParam14=1011,
      ZSTD_c_experimentalParam15=1012,
-     ZSTD_c_experimentalParam16=1013
+     ZSTD_c_experimentalParam16=1013,
+     ZSTD_c_experimentalParam17=1014,
+     ZSTD_c_experimentalParam18=1015
 } ZSTD_cParameter;
 

    
 
    typedef struct {
@@ -444,7 +470,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  /* accept NULL pointer */
                   They will be used to compress next frame.
                   Resetting session never fails.
   - The parameters : changes all parameters back to "default".
-                  This removes any reference to any dictionary too.
+                  This also removes any reference to any dictionary or external matchfinder.
                   Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
                   otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
   - Both : similar to resetting the session, followed by resetting parameters.
@@ -483,13 +509,15 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  /* accept NULL pointer */
      * ZSTD_d_stableOutBuffer
      * ZSTD_d_forceIgnoreChecksum
      * ZSTD_d_refMultipleDDicts
+     * ZSTD_d_disableHuffmanAssembly
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
      ZSTD_d_experimentalParam1=1000,
      ZSTD_d_experimentalParam2=1001,
      ZSTD_d_experimentalParam3=1002,
-     ZSTD_d_experimentalParam4=1003
+     ZSTD_d_experimentalParam4=1003,
+     ZSTD_d_experimentalParam5=1004
 
 } ZSTD_dParameter;
 

    
@@ -658,6 +686,9 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
      ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+
+ Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
+ to compress with a dictionary.
  
 

    
 
@@ -693,6 +724,33 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 size_t ZSTD_freeDStream(ZSTD_DStream* zds);      /* accept NULL pointer */
 

    
 
    Streaming decompression functions

    
+
    size_t ZSTD_initDStream(ZSTD_DStream* zds);
+
     Initialize/reset DStream state for new decompression operation.
+ Call before new decompression operation using same DStream.
+
+ Note : This function is redundant with the advanced API and equivalent to:
+     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+     ZSTD_DCtx_refDDict(zds, NULL);
+ 
+

    
+
+
    size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
     Streaming decompression function.
+ Call repetitively to consume full input updating it as necessary.
+ Function will update both input and output `pos` fields exposing current state via these fields:
+ - `input.pos < input.size`, some input remaining and caller should provide remaining input
+   on the next call.
+ - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
+ - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
+   call ZSTD_decompressStream() again to flush remaining data to output.
+ Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
+
+ @return : 0 when a frame is completely decoded and fully flushed,
+           or an error code, which can be tested using ZSTD_isError(),
+           or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
+ 
+

    
+
 
    size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
 

    
 
    size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
@@ -818,8 +876,9 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  /* accept NULL pointer */
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
            meaning "return to no-dictionary mode".
-  Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
-           To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
+  Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
+           until parameters are reset, a new dictionary is loaded, or the dictionary
+           is explicitly invalidated by loading a NULL dictionary.
   Note 2 : Loading a dictionary involves building tables.
            It's also a CPU consuming operation, with non-negligible impact on latency.
            Tables are dependent on compression parameters, and for this reason,
@@ -832,7 +891,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  /* accept NULL pointer */
 

    
 
 
    size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
-
      Reference a prepared dictionary, to be used for all next compressed frames.
+
      Reference a prepared dictionary, to be used for all future compressed frames.
   Note that compression parameters are enforced from within CDict,
   and supersede any compression parameter previously set within CCtx.
   The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
@@ -867,9 +926,9 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  /* accept NULL pointer */
 

    
 
 
    size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-
      Create an internal DDict from dict buffer,
-  to be used to decompress next frames.
-  The dictionary remains valid for all future frames, until explicitly invalidated.
+
      Create an internal DDict from dict buffer, to be used to decompress all future frames.
+  The dictionary remains valid for all future frames, until explicitly invalidated, or
+  a new dictionary is loaded.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
             meaning "return to no-dictionary mode".
@@ -893,9 +952,10 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  /* accept NULL pointer */
   The memory for the table is allocated on the first call to refDDict, and can be
   freed with ZSTD_freeDCtx().
 
+  If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
+  will be managed, and referencing a dictionary effectively "discards" any previous one.
+
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
-  Note 1 : Currently, only one dictionary can be managed.
-           Referencing a new dictionary effectively "discards" any previous one.
   Special: referencing a NULL DDict means "return to no-dictionary mode".
   Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
  
@@ -1131,11 +1191,65 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
            or an error code (if srcSize is too small) 
 

    
 
+
    ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
+
     Zstd supports in-place decompression, where the input and output buffers overlap.
+ In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
+ and the input buffer must be at the end of the output buffer.
+
+  _______________________ Output Buffer ________________________
+ |                                                              |
+ |                                        ____ Input Buffer ____|
+ |                                       |                      |
+ v                                       v                      v
+ |---------------------------------------|-----------|----------|
+ ^                                                   ^          ^
+ |___________________ Output_Size ___________________|_ Margin _|
+
+ NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
+ NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
+ ZSTD_decompressDCtx().
+ NOTE: This function supports multi-frame input.
+
+ @param src The compressed frame(s)
+ @param srcSize The size of the compressed frame(s)
+ @returns The decompression margin or an error that can be checked with ZSTD_isError().
+ 
+

    
+
+
    #define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)(                                              \
+        ZSTD_FRAMEHEADERSIZE_MAX                                                              /* Frame header */ + \
+        4                                                                                         /* checksum */ + \
+        ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
+        (blockSize)                                                                    /* One block of margin */   \
+    ))
+
     Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
+ the compressed frame, compute it from the original size and the blockSizeLog.
+ See ZSTD_decompressionMargin() for details.
+
+ WARNING: This macro does not support multi-frame input, the input must be a single
+ zstd frame. If you need that support use the function, or implement it yourself.
+
+ @param originalSize The original uncompressed size of the data.
+ @param blockSize    The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
+                     Unless you explicitly set the windowLog smaller than
+                     ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
+ 
+

    
+
 
    typedef enum {
   ZSTD_sf_noBlockDelimiters = 0,         /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
   ZSTD_sf_explicitBlockDelimiters = 1    /* Representation of ZSTD_Sequence contains explicit block delimiters */
 } ZSTD_sequenceFormat_e;
 

    
+
    ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
+
     `srcSize` : size of the input buffer
+  @return : upper-bound for the number of sequences that can be generated
+            from a buffer of srcSize bytes
+
+  note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
+ 
+

    
+
 
     Generate sequences using ZSTD_compress2(), given a source buffer.
 
  Each block will end with a dummy sequence
@@ -1260,8 +1374,11 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
   and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
   Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
 
-  Note 2 : only single-threaded compression is supported.
+  Note : only single-threaded compression is supported.
   ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+
+  Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the external matchfinder API at this time.
+  Size estimates assume that no external matchfinder is registered.
  
 

    
 
@@ -1281,7 +1398,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, si
   or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
   Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
          an internal ?Dict will be created, which additional size is not estimated here.
-         In this case, get total size by adding ZSTD_estimate?DictSize 
+         In this case, get total size by adding ZSTD_estimate?DictSize
+  Note 2 : only single-threaded compression is supported.
+  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+  Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the external matchfinder API at this time.
+  Size estimates assume that no external matchfinder is registered.
+ 
 

    
 
 
    ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
@@ -1381,20 +1503,27 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPo
   This function never fails (wide contract) 
 

    
 
-
    ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_compress2")
+
    ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
+
      Set all parameters provided within @cparams into the working @cctx.
+  Note : if modifying parameters during compression (MT mode only),
+         note that changes to the .windowLog parameter will be ignored.
+ @return 0 on success, or an error code (can be checked with ZSTD_isError()) 
+

    
+
+
    ZSTD_DEPRECATED("use ZSTD_compress2")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
-                              void* dst, size_t dstCapacity,
-                        const void* src, size_t srcSize,
-                        const void* dict,size_t dictSize,
-                              ZSTD_parameters params);
+                  void* dst, size_t dstCapacity,
+            const void* src, size_t srcSize,
+            const void* dict,size_t dictSize,
+                  ZSTD_parameters params);
 
      Note : this function is now DEPRECATED.
          It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
   This prototype will generate compilation warnings. 
 

    
 
-
    ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+
    ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
                                   void* dst, size_t dstCapacity,
                             const void* src, size_t srcSize,
@@ -1555,8 +1684,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);  /*
  
 

    
 
-
    ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+
    ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+ZSTDLIB_STATIC_API
 size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
 
      This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
   Instruct the decoder context about what kind of data to decode next.
@@ -1582,8 +1711,8 @@ size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
 
    
 
 
    Advanced Streaming compression functions

    
-
    ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+
    ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
              int compressionLevel,
              unsigned long long pledgedSrcSize);
@@ -1600,8 +1729,8 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  
 

    
 
-
    ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+
    ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
          const void* dict, size_t dictSize,
                int compressionLevel);
@@ -1618,8 +1747,8 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
  
 

    
 
-
    ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+
    ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
         const void* dict, size_t dictSize,
               ZSTD_parameters params,
@@ -1640,8 +1769,8 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
  
 

    
 
-
    ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+
    ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
 
     This function is DEPRECATED, and equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -1652,8 +1781,8 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
  
 

    
 
-
    ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+
    ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
                    const ZSTD_CDict* cdict,
                          ZSTD_frameParameters fParams,
@@ -1674,8 +1803,8 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
  
 

    
 
-
    ZSTDLIB_STATIC_API
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+
    ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 
     This function is DEPRECATED, and is equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -1721,32 +1850,32 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 

    
 
 
    Advanced Streaming decompression functions

    
-
    ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+
    ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 
    
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
      ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
 
  note: no dictionary will be used if dict == NULL or dictSize < 8
- Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 

    
 
-
    ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+
    ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
 
    
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
      ZSTD_DCtx_refDDict(zds, ddict);
 
  note : ddict is referenced, it must outlive decompression session
- Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 

    
 
-
    ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
+
    ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 
    
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
 
  re-use decompression parameters from previous init; saves dictionary loading
- Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 

    
 
@@ -1764,7 +1893,6 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 
   Start by initializing a context.
   Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
-  It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
 
   Then, consume your input using ZSTD_compressContinue().
   There are some important considerations to keep in mind when using this advanced function :
@@ -1788,8 +1916,9 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 
    Buffer-less streaming compression functions
    ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
-ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 

    
+
    size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
+

    
 
    size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
 

    
 
    Buffer-less streaming decompression (synchronous mode)
    @@ -1801,8 +1930,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* prepar
   Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
   Data fragment must be large enough to ensure successful decoding.
  `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
-  @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
-           >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
+  result  : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
+           >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
            errorCode, which can be tested using ZSTD_isError().
 
   It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
@@ -1821,7 +1950,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* prepar
 
   The most memory efficient way is to use a round buffer of sufficient size.
   Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
-  which can @return an error code if required value is too large for current system (in 32-bits mode).
+  which can return an error code if required value is too large for current system (in 32-bits mode).
   In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
   up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
   which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
@@ -1841,7 +1970,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* prepar
   ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
   ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
 
- @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+  result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
   It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
   It can also be an error code, which can be tested with ZSTD_isError().
 
@@ -1872,6 +2001,8 @@ typedef struct {
     unsigned headerSize;
     unsigned dictID;
     unsigned checksumFlag;
+    unsigned _reserved1;
+    unsigned _reserved2;
 } ZSTD_frameHeader;
 

    
 
    ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   /**< doesn't consume input */
@@ -1899,7 +2030,6 @@ ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowS
     - It is necessary to init context before starting
       + compression : any ZSTD_compressBegin*() variant, including with dictionary
       + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
-      + copyCCtx() and copyDCtx() can be used too
     - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
       + If input is larger than a block size, it's necessary to split input data into multiple blocks
       + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
@@ -1920,5 +2050,33 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_
 ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
 

    
+
    ZSTDLIB_STATIC_API void
+ZSTD_registerExternalMatchFinder(
+  ZSTD_CCtx* cctx,
+  void* externalMatchState,
+  ZSTD_externalMatchFinder_F* externalMatchFinder
+);
+
     Instruct zstd to use an external matchfinder function.
+
+ The externalMatchState must be initialized by the caller, and the caller is
+ responsible for managing its lifetime. This parameter is sticky across
+ compressions. It will remain set until the user explicitly resets compression
+ parameters.
+
+ External matchfinder registration is considered to be an "advanced parameter",
+ part of the "advanced API". This means it will only have an effect on
+ compression APIs which respect advanced parameters, such as compress2() and
+ compressStream(). Older compression APIs such as compressCCtx(), which predate
+ the introduction of "advanced parameters", will ignore any external matchfinder
+ setting.
+
+ The external matchfinder can be "cleared" by registering a NULL external
+ matchfinder function pointer. This removes all limitations described above in
+ the "LIMITATIONS" section of the API docs.
+
+ The user is strongly encouraged to read the full API documentation (above)
+ before calling this function. 
+

    
+
 
 
diff --git a/examples/Makefile b/examples/Makefile
index 8d7361dd8..31f52d357 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/common.h b/examples/common.h
index 9b09030eb..4873e877a 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c
index 7c7d34288..83edc1cad 100644
--- a/examples/dictionary_compression.c
+++ b/examples/dictionary_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c
index 107cfc1ee..e6c999964 100644
--- a/examples/dictionary_decompression.c
+++ b/examples/dictionary_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/multiple_simple_compression.c b/examples/multiple_simple_compression.c
index 5d2a28fcd..bf77ca133 100644
--- a/examples/multiple_simple_compression.c
+++ b/examples/multiple_simple_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
index d4efc8e57..b12ad03dc 100644
--- a/examples/multiple_streaming_compression.c
+++ b/examples/multiple_streaming_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/simple_compression.c b/examples/simple_compression.c
index 27a65b17f..7c880725f 100644
--- a/examples/simple_compression.c
+++ b/examples/simple_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c
index 59c1fd414..f499156f6 100644
--- a/examples/simple_decompression.c
+++ b/examples/simple_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c
index ff1875829..ed0a3a69c 100644
--- a/examples/streaming_compression.c
+++ b/examples/streaming_compression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_compression_thread_pool.c b/examples/streaming_compression_thread_pool.c
index 21cb3d549..a1a024129 100644
--- a/examples/streaming_compression_thread_pool.c
+++ b/examples/streaming_compression_thread_pool.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Martin Liska, SUSE, Facebook, Inc.
+ * Copyright (c) Martin Liska, SUSE, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c
index 6dc4c2267..95fa11227 100644
--- a/examples/streaming_decompression.c
+++ b/examples/streaming_decompression.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/examples/streaming_memory_usage.c b/examples/streaming_memory_usage.c
index a5219ef1e..957acb61a 100644
--- a/examples/streaming_memory_usage.c
+++ b/examples/streaming_memory_usage.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/Makefile b/lib/Makefile
index 74c6f1baf..a4cf61ab1 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/README.md b/lib/README.md
index 015cde567..c3b5d1817 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -161,6 +161,13 @@ The file structure is designed to make this selection manually achievable for an
   `ZSTD_DCtx` decompression contexts,
   but might also result in a small decompression speed cost.
 
+- The C compiler macros `ZSTDLIB_VISIBLE`, `ZSTDERRORLIB_VISIBLE` and `ZDICTLIB_VISIBLE`
+  can be overridden to control the visibility of zstd's API. Additionally,
+  `ZSTDLIB_STATIC_API` and `ZDICTLIB_STATIC_API` can be overridden to control the visibility
+  of zstd's static API. Specifically, it can be set to `ZSTDLIB_HIDDEN` to hide the symbols
+  from the shared library. These macros default to `ZSTDLIB_VISIBILITY`,
+  `ZSTDERRORLIB_VSIBILITY`, and `ZDICTLIB_VISIBILITY` if unset, for backwards compatibility
+  with the old macro names.
 
 #### Windows : using MinGW+MSYS to create DLL
 
diff --git a/lib/common/bits.h b/lib/common/bits.h
index c0e917750..7939f3d0f 100644
--- a/lib/common/bits.h
+++ b/lib/common/bits.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
index 841778626..db1b4cf13 100644
--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * bitstream
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index 42f289e0b..d4f2f285d 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -281,7 +281,18 @@
 *  Sanitizer
 *****************************************************************/
 
-#if ZSTD_MEMORY_SANITIZER
+/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an
+ * abundance of caution, disable our custom poisoning on mingw. */
+#ifdef __MINGW32__
+#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE
+#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1
+#endif
+#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE
+#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1
+#endif
+#endif
+
+#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE)
 /* Not all platforms that support msan provide sanitizers/msan_interface.h.
  * We therefore declare the functions we need ourselves, rather than trying to
  * include the header file... */
@@ -302,7 +313,7 @@ void __msan_poison(const volatile void *a, size_t size);
 intptr_t __msan_test_shadow(const volatile void *x, size_t size);
 #endif
 
-#if ZSTD_ADDRESS_SANITIZER
+#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE)
 /* Not all platforms that support asan provide sanitizers/asan_interface.h.
  * We therefore declare the functions we need ourselves, rather than trying to
  * include the header file... */
diff --git a/lib/common/cpu.h b/lib/common/cpu.h
index 8acd33be3..8bc34a36d 100644
--- a/lib/common/cpu.h
+++ b/lib/common/cpu.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/debug.c b/lib/common/debug.c
index bb863c9ea..ebf7bfccf 100644
--- a/lib/common/debug.c
+++ b/lib/common/debug.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/debug.h b/lib/common/debug.h
index 3b2a320a1..0e9817ea6 100644
--- a/lib/common/debug.h
+++ b/lib/common/debug.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c
index 98bd4238d..e2173afb0 100644
--- a/lib/common/entropy_common.c
+++ b/lib/common/entropy_common.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * Common functions of New Generation Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -19,7 +19,6 @@
 #include "error_private.h"       /* ERR_*, ERROR */
 #define FSE_STATIC_LINKING_ONLY  /* FSE_MIN_TABLELOG */
 #include "fse.h"
-#define HUF_STATIC_LINKING_ONLY  /* HUF_TABLELOG_ABSOLUTEMAX */
 #include "huf.h"
 #include "bits.h"                /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */
 
@@ -237,7 +236,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
                      const void* src, size_t srcSize)
 {
     U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
-    return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0);
+    return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0);
 }
 
 FORCE_INLINE_TEMPLATE size_t
@@ -329,13 +328,13 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
                      U32* nbSymbolsPtr, U32* tableLogPtr,
                      const void* src, size_t srcSize,
                      void* workSpace, size_t wkspSize,
-                     int bmi2)
+                     int flags)
 {
 #if DYNAMIC_BMI2
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
         return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
     }
 #endif
-    (void)bmi2;
+    (void)flags;
     return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
 }
diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index 1b67500f3..0cff6b80b 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -29,7 +29,9 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
     case PREFIX(corruption_detected): return "Data corruption detected";
     case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+    case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification";
     case PREFIX(parameter_unsupported): return "Unsupported parameter";
+    case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters";
     case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
     case PREFIX(init_missing): return "Context should be init first";
     case PREFIX(memory_allocation): return "Allocation error : not enough memory";
@@ -45,11 +47,15 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
     case PREFIX(srcSize_wrong): return "Src size is incorrect";
     case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
+    case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full";
+    case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty";
         /* following error codes are not stable and may be removed or changed in a future version */
     case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
     case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
     case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
     case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
+    case PREFIX(externalMatchFinder_failed): return "External matchfinder returned an error code";
+    case PREFIX(externalSequences_invalid): return "External sequences are not valid";
     case PREFIX(maxCode):
     default: return notErrorCode;
     }
diff --git a/lib/common/error_private.h b/lib/common/error_private.h
index 007d81066..325daad40 100644
--- a/lib/common/error_private.h
+++ b/lib/common/error_private.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/fse.h b/lib/common/fse.h
index 466a07281..02a1f0bc5 100644
--- a/lib/common/fse.h
+++ b/lib/common/fse.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * FSE : Finite State Entropy codec
  * Public Prototypes declaration
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -53,34 +53,6 @@ extern "C" {
 FSE_PUBLIC_API unsigned FSE_versionNumber(void);   /**< library version number; to be used when checking dll version */
 
 
-/*-****************************************
-*  FSE simple functions
-******************************************/
-/*! FSE_compress() :
-    Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
-    'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
-    @return : size of compressed data (<= dstCapacity).
-    Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
-                     if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
-                     if FSE_isError(return), compression failed (more details using FSE_getErrorName())
-*/
-FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
-                             const void* src, size_t srcSize);
-
-/*! FSE_decompress():
-    Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
-    into already allocated destination buffer 'dst', of size 'dstCapacity'.
-    @return : size of regenerated data (<= maxDstSize),
-              or an error code, which can be tested using FSE_isError() .
-
-    ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
-    Why ? : making this distinction requires a header.
-    Header management is intentionally delegated to the user layer, which can better manage special cases.
-*/
-FSE_PUBLIC_API size_t FSE_decompress(void* dst,  size_t dstCapacity,
-                               const void* cSrc, size_t cSrcSize);
-
-
 /*-*****************************************
 *  Tool functions
 ******************************************/
@@ -91,20 +63,6 @@ FSE_PUBLIC_API unsigned    FSE_isError(size_t code);        /* tells if a return
 FSE_PUBLIC_API const char* FSE_getErrorName(size_t code);   /* provides error code string (useful for debugging) */
 
 
-/*-*****************************************
-*  FSE advanced functions
-******************************************/
-/*! FSE_compress2() :
-    Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
-    Both parameters can be defined as '0' to mean : use default value
-    @return : size of compressed data
-    Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
-                     if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
-                     if FSE_isError(return), it's an error code.
-*/
-FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-
-
 /*-*****************************************
 *  FSE detailed API
 ******************************************/
@@ -164,8 +122,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
 /*! Constructor and Destructor of FSE_CTable.
     Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
 typedef unsigned FSE_CTable;   /* don't allocate that. It's only meant to be more restrictive than void* */
-FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
-FSE_PUBLIC_API void        FSE_freeCTable (FSE_CTable* ct);
 
 /*! FSE_buildCTable():
     Builds `ct`, which must be already allocated, using FSE_createCTable().
@@ -241,23 +197,7 @@ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
                            unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
                            const void* rBuffer, size_t rBuffSize, int bmi2);
 
-/*! Constructor and Destructor of FSE_DTable.
-    Note that its size depends on 'tableLog' */
 typedef unsigned FSE_DTable;   /* don't allocate that. It's just a way to be more restrictive than void* */
-FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
-FSE_PUBLIC_API void        FSE_freeDTable(FSE_DTable* dt);
-
-/*! FSE_buildDTable():
-    Builds 'dt', which must be already allocated, using FSE_createDTable().
-    return : 0, or an errorCode, which can be tested using FSE_isError() */
-FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
-
-/*! FSE_decompress_usingDTable():
-    Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
-    into `dst` which must be already allocated.
-    @return : size of regenerated data (necessarily <= `dstCapacity`),
-              or an errorCode, which can be tested using FSE_isError() */
-FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
 
 /*!
 Tutorial :
@@ -320,16 +260,6 @@ If there is an error, the function will return an error code, which can be teste
 unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
 /**< same as FSE_optimalTableLog(), which used `minus==2` */
 
-/* FSE_compress_wksp() :
- * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
- * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
- */
-#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue)   ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
-size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
-
-size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
-/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
-
 size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
 /**< build a fake FSE_CTable, designed to compress always the same symbolValue */
 
@@ -347,19 +277,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
 FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
 /**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
 
-size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
-/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
-
-size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
-/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
-
 #define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
 #define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
-size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize);
-/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */
-
 size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
-/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */
+/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`.
+ * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */
 
 typedef enum {
    FSE_repeat_none,  /**< Cannot use the previous table */
diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c
index 7034fd97b..1e1c9f92d 100644
--- a/lib/common/fse_decompress.c
+++ b/lib/common/fse_decompress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * FSE : Finite State Entropy decoder
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -56,19 +56,6 @@
 #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
 #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
 
-
-/* Function templates */
-FSE_DTable* FSE_createDTable (unsigned tableLog)
-{
-    if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
-    return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
-}
-
-void FSE_freeDTable (FSE_DTable* dt)
-{
-    ZSTD_free(dt);
-}
-
 static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
 {
     void* const tdPtr = dt+1;   /* because *dt is unsigned, 32-bits aligned on 32-bits */
@@ -185,49 +172,6 @@ size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsi
 /*-*******************************************************
 *  Decompression (Byte symbols)
 *********************************************************/
-size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
-{
-    void* ptr = dt;
-    FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
-    void* dPtr = dt + 1;
-    FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
-
-    DTableH->tableLog = 0;
-    DTableH->fastMode = 0;
-
-    cell->newState = 0;
-    cell->symbol = symbolValue;
-    cell->nbBits = 0;
-
-    return 0;
-}
-
-
-size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
-{
-    void* ptr = dt;
-    FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
-    void* dPtr = dt + 1;
-    FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
-    const unsigned tableSize = 1 << nbBits;
-    const unsigned tableMask = tableSize - 1;
-    const unsigned maxSV1 = tableMask+1;
-    unsigned s;
-
-    /* Sanity checks */
-    if (nbBits < 1) return ERROR(GENERIC);         /* min size */
-
-    /* Build Decoding Table */
-    DTableH->tableLog = (U16)nbBits;
-    DTableH->fastMode = 1;
-    for (s=0; sfastMode;
-
-    /* select fast mode (static) */
-    if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
-    return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
-}
-
-
-size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
-{
-    return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
 typedef struct {
     short ncount[FSE_MAX_SYMBOL_VALUE + 1];
     FSE_DTable dtable[1]; /* Dynamically sized */
@@ -384,22 +308,4 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc,
     return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
 }
 
-
-typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
-
-#ifndef ZSTD_NO_UNUSED_FUNCTIONS
-size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) {
-    U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)];
-    return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp));
-}
-
-size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
-{
-    /* Static analyzer seems unable to understand this table will be properly initialized later */
-    U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
-    return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp));
-}
-#endif
-
-
 #endif   /* FSE_COMMONDEFS_ONLY */
diff --git a/lib/common/huf.h b/lib/common/huf.h
index 595b2f6db..73d1ee565 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * huff0 huffman codec,
  * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -21,100 +21,23 @@ extern "C" {
 
 /* *** Dependencies *** */
 #include "zstd_deps.h"    /* size_t */
-
-
-/* *** library symbols visibility *** */
-/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual,
- *        HUF symbols remain "private" (internal symbols for library only).
- *        Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */
-#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
-#  define HUF_PUBLIC_API __attribute__ ((visibility ("default")))
-#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1)   /* Visual expected */
-#  define HUF_PUBLIC_API __declspec(dllexport)
-#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
-#  define HUF_PUBLIC_API __declspec(dllimport)  /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */
-#else
-#  define HUF_PUBLIC_API
-#endif
-
-
-/* ========================== */
-/* ***  simple functions  *** */
-/* ========================== */
-
-/** HUF_compress() :
- *  Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
- * 'dst' buffer must be already allocated.
- *  Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
- * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
- * @return : size of compressed data (<= `dstCapacity`).
- *  Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
- *                   if HUF_isError(return), compression failed (more details using HUF_getErrorName())
- */
-HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
-                             const void* src, size_t srcSize);
-
-/** HUF_decompress() :
- *  Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
- *  into already allocated buffer 'dst', of minimum size 'dstSize'.
- * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
- *  Note : in contrast with FSE, HUF_decompress can regenerate
- *         RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
- *         because it knows size to regenerate (originalSize).
- * @return : size of regenerated data (== originalSize),
- *           or an error code, which can be tested using HUF_isError()
- */
-HUF_PUBLIC_API size_t HUF_decompress(void* dst,  size_t originalSize,
-                               const void* cSrc, size_t cSrcSize);
-
-
-/* ***   Tool functions *** */
-#define HUF_BLOCKSIZE_MAX (128 * 1024)                  /**< maximum input size for a single block compressed with HUF_compress */
-HUF_PUBLIC_API size_t HUF_compressBound(size_t size);   /**< maximum compressed size (worst case) */
-
-/* Error Management */
-HUF_PUBLIC_API unsigned    HUF_isError(size_t code);       /**< tells if a return value is an error code */
-HUF_PUBLIC_API const char* HUF_getErrorName(size_t code);  /**< provides error code string (useful for debugging) */
-
-
-/* ***   Advanced function   *** */
-
-/** HUF_compress2() :
- *  Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
- * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
- * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
-HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
-                               const void* src, size_t srcSize,
-                               unsigned maxSymbolValue, unsigned tableLog);
-
-/** HUF_compress4X_wksp() :
- *  Same as HUF_compress2(), but uses externally allocated `workSpace`.
- * `workspace` must be at least as large as HUF_WORKSPACE_SIZE */
-#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
-#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
-HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
-                                     unsigned maxSymbolValue, unsigned tableLog,
-                                     void* workSpace, size_t wkspSize);
-
-#endif   /* HUF_H_298734234 */
-
-/* ******************************************************************
- *  WARNING !!
- *  The following section contains advanced and experimental definitions
- *  which shall never be used in the context of a dynamic library,
- *  because they are not guaranteed to remain stable in the future.
- *  Only consider them in association with static linking.
- * *****************************************************************/
-#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
-#define HUF_H_HUF_STATIC_LINKING_ONLY
-
-/* *** Dependencies *** */
-#include "mem.h"   /* U32 */
+#include "mem.h"          /* U32 */
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
 
 
+/* ***   Tool functions *** */
+#define HUF_BLOCKSIZE_MAX (128 * 1024)   /**< maximum input size for a single block compressed with HUF_compress */
+size_t HUF_compressBound(size_t size);   /**< maximum compressed size (worst case) */
+
+/* Error Management */
+unsigned    HUF_isError(size_t code);       /**< tells if a return value is an error code */
+const char* HUF_getErrorName(size_t code);  /**< provides error code string (useful for debugging) */
+
+
+#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
+#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
+
 /* *** Constants *** */
 #define HUF_TABLELOG_MAX      12      /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
 #define HUF_TABLELOG_DEFAULT  11      /* default tableLog value when none specified */
@@ -154,30 +77,49 @@ typedef U32 HUF_DTable;
 /* ****************************************
 *  Advanced decompression functions
 ******************************************/
-size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
-#endif
 
-size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< decodes RLE and uncompressed */
-size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
-size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
-size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
-size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< single-symbol decoder */
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
-size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< double-symbols decoder */
-#endif
+/**
+ * Huffman flags bitset.
+ * For all flags, 0 is the default value.
+ */
+typedef enum {
+    /**
+     * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime.
+     * Otherwise: Ignored.
+     */
+    HUF_flags_bmi2 = (1 << 0),
+    /**
+     * If set: Test possible table depths to find the one that produces the smallest header + encoded size.
+     * If unset: Use heuristic to find the table depth.
+     */
+    HUF_flags_optimalDepth = (1 << 1),
+    /**
+     * If set: If the previous table can encode the input, always reuse the previous table.
+     * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output.
+     */
+    HUF_flags_preferRepeat = (1 << 2),
+    /**
+     * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress.
+     * If unset: Always histogram the entire input.
+     */
+    HUF_flags_suspectUncompressible = (1 << 3),
+    /**
+     * If set: Don't use assembly implementations
+     * If unset: Allow using assembly implementations
+     */
+    HUF_flags_disableAsm = (1 << 4),
+    /**
+     * If set: Don't use the fast decoding loop, always use the fallback decoding loop.
+     * If unset: Use the fast decoding loop when possible.
+     */
+    HUF_flags_disableFast = (1 << 5)
+} HUF_flags_e;
 
 
 /* ****************************************
  *  HUF detailed API
  * ****************************************/
 #define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
-typedef enum {
-   HUF_depth_fast, /** Use heuristic to find the table depth**/
-   HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/
- } HUF_depth_mode;
 
 /*! HUF_compress() does the following:
  *  1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
@@ -193,12 +135,9 @@ typedef enum {
 unsigned HUF_minTableLog(unsigned symbolCardinality);
 unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue);
 unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace,
- size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode); /* table is used as scratch space for building and testing tables, not a return value */
-size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);   /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
-size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
+ size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */
 size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
-size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
-size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
 size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
 int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
 
@@ -218,14 +157,13 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
                        const void* src, size_t srcSize,
                        unsigned maxSymbolValue, unsigned tableLog,
                        void* workSpace, size_t wkspSize,    /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
-                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
-                       unsigned suspectUncompressible, HUF_depth_mode depthMode);
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
 
 /** HUF_buildCTable_wksp() :
  *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
  * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
  */
-#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
+#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192)
 #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
 size_t HUF_buildCTable_wksp (HUF_CElt* tree,
                        const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
@@ -251,7 +189,7 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
                           U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
                           const void* src, size_t srcSize,
                           void* workspace, size_t wkspSize,
-                          int bmi2);
+                          int flags);
 
 /** HUF_readCTable() :
  *  Loading a CTable saved with HUF_writeCTable() */
@@ -289,32 +227,12 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
 #define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
 #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
 
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
-size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
-size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
-#endif
-
-size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-
 
 /* ====================== */
 /* single stream variants */
 /* ====================== */
 
-size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 */
-size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
-size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
 /** HUF_compress1X_repeat() :
  *  Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
  *  If it uses hufTable it does not modify hufTable or repeat.
@@ -325,50 +243,30 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
                        const void* src, size_t srcSize,
                        unsigned maxSymbolValue, unsigned tableLog,
                        void* workSpace, size_t wkspSize,   /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
-                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
-                       unsigned suspectUncompressible, HUF_depth_mode depthMode);
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
 
-size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
 #ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */
-#endif
-
-size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
-size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
-size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< single-symbol decoder */
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
-size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< double-symbols decoder */
-#endif
-
-size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);   /**< automatic selection of sing or double symbol decoder, based on DTable */
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);   /**< double-symbols decoder */
 #endif
 
 /* BMI2 variants.
  * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
  */
-size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
 #ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
 #endif
-size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
-size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
 #ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
 #endif
 #ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
 #endif
 
-#endif /* HUF_STATIC_LINKING_ONLY */
+#endif   /* HUF_H_298734234 */
 
 #if defined (__cplusplus)
 }
diff --git a/lib/common/mem.h b/lib/common/mem.h
index 4b10f7c1f..98dd47a04 100644
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -133,21 +133,15 @@ MEM_STATIC size_t MEM_swapST(size_t in);
 /*-**************************************************************
 *  Memory I/O Implementation
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
- *            In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
+ * Default  : method 1 if supported, else method 0
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
+#  ifdef __GNUC__
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
@@ -190,30 +184,19 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
-    __pragma( pack(push, 1) )
-    typedef struct { U16 v; } unalign16;
-    typedef struct { U32 v; } unalign32;
-    typedef struct { U64 v; } unalign64;
-    typedef struct { size_t v; } unalignArch;
-    __pragma( pack(pop) )
-#else
-    typedef struct { U16 v; } __attribute__((packed)) unalign16;
-    typedef struct { U32 v; } __attribute__((packed)) unalign32;
-    typedef struct { U64 v; } __attribute__((packed)) unalign64;
-    typedef struct { size_t v; } __attribute__((packed)) unalignArch;
-#endif
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
+typedef __attribute__((aligned(1))) size_t unalignArch;
 
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
-MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }
+MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
+MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; }
 
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; }
 
 #else
 
diff --git a/lib/common/pool.c b/lib/common/pool.c
index bf21c57ed..f3d9d0854 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -173,7 +173,7 @@ static void POOL_join(POOL_ctx* ctx) {
     /* Join all of the threads */
     {   size_t i;
         for (i = 0; i < ctx->threadCapacity; ++i) {
-            ZSTD_pthread_join(ctx->threads[i], NULL);  /* note : could fail */
+            ZSTD_pthread_join(ctx->threads[i]);  /* note : could fail */
     }   }
 }
 
@@ -271,7 +271,9 @@ static int isQueueFull(POOL_ctx const* ctx) {
 static void
 POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
 {
-    POOL_job const job = {function, opaque};
+    POOL_job job;
+    job.function = function;
+    job.opaque = opaque;
     assert(ctx != NULL);
     if (ctx->shutdown) return;
 
diff --git a/lib/common/pool.h b/lib/common/pool.h
index b86a3452e..eb22ff509 100644
--- a/lib/common/pool.h
+++ b/lib/common/pool.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/portability_macros.h b/lib/common/portability_macros.h
index 1650fa3d8..8fd6ea82d 100644
--- a/lib/common/portability_macros.h
+++ b/lib/common/portability_macros.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -137,12 +137,20 @@
 /*
  * For x86 ELF targets, add .note.gnu.property section for Intel CET in
  * assembly sources when CET is enabled.
+ *
+ * Additionally, any function that may be called indirectly must begin
+ * with ZSTD_CET_ENDBRANCH.
  */
 #if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \
     && defined(__has_include)
 # if __has_include()
 #  include 
+#  define ZSTD_CET_ENDBRANCH _CET_ENDBR
 # endif
 #endif
 
+#ifndef ZSTD_CET_ENDBRANCH
+# define ZSTD_CET_ENDBRANCH
+#endif
+
 #endif /* ZSTD_PORTABILITY_MACROS_H */
diff --git a/lib/common/threading.c b/lib/common/threading.c
index 4f69f7662..f2341105a 100644
--- a/lib/common/threading.c
+++ b/lib/common/threading.c
@@ -23,8 +23,7 @@ int g_ZSTD_threading_useless_symbol;
 #if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
 
 /**
- * Windows minimalist Pthread Wrapper, based on :
- * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ * Windows minimalist Pthread Wrapper
  */
 
 
@@ -35,39 +34,92 @@ int g_ZSTD_threading_useless_symbol;
 
 /* ===  Implementation  === */
 
+typedef struct {
+    void* (*start_routine)(void*);
+    void* arg;
+    int initialized;
+    ZSTD_pthread_cond_t initialized_cond;
+    ZSTD_pthread_mutex_t initialized_mutex;
+} ZSTD_thread_params_t;
+
 static unsigned __stdcall worker(void *arg)
 {
-    ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
-    thread->arg = thread->start_routine(thread->arg);
+    void* (*start_routine)(void*);
+    void* thread_arg;
+
+    /* Inialized thread_arg and start_routine and signal main thread that we don't need it
+     * to wait any longer.
+     */
+    {
+        ZSTD_thread_params_t*  thread_param = (ZSTD_thread_params_t*)arg;
+        thread_arg = thread_param->arg;
+        start_routine = thread_param->start_routine;
+
+        /* Signal main thread that we are running and do not depend on its memory anymore */
+        ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex);
+        thread_param->initialized = 1;
+        ZSTD_pthread_cond_signal(&thread_param->initialized_cond);
+        ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex);
+    }
+
+    start_routine(thread_arg);
+
     return 0;
 }
 
 int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
             void* (*start_routine) (void*), void* arg)
 {
+    ZSTD_thread_params_t thread_param;
     (void)unused;
-    thread->arg = arg;
-    thread->start_routine = start_routine;
-    thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
 
-    if (!thread->handle)
+    thread_param.start_routine = start_routine;
+    thread_param.arg = arg;
+    thread_param.initialized = 0;
+    *thread = NULL;
+
+    /* Setup thread initialization synchronization */
+    if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) {
+        /* Should never happen on Windows */
+        return -1;
+    }
+    if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) {
+        /* Should never happen on Windows */
+        ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
+        return -1;
+    }
+
+    /* Spawn thread */
+    *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL);
+    if (!thread) {
+        ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex);
+        ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
         return errno;
-    else
-        return 0;
+    }
+
+    /* Wait for thread to be initialized */
+    ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex);
+    while(!thread_param.initialized) {
+        ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex);
+    }
+    ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex);
+    ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex);
+    ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
+
+    return 0;
 }
 
-int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
+int ZSTD_pthread_join(ZSTD_pthread_t thread)
 {
     DWORD result;
 
-    if (!thread.handle) return 0;
+    if (!thread) return 0;
 
-    result = WaitForSingleObject(thread.handle, INFINITE);
-    CloseHandle(thread.handle);
+    result = WaitForSingleObject(thread, INFINITE);
+    CloseHandle(thread);
 
     switch (result) {
     case WAIT_OBJECT_0:
-        if (value_ptr) *value_ptr = thread.arg;
         return 0;
     case WAIT_ABANDONED:
         return EINVAL;
diff --git a/lib/common/threading.h b/lib/common/threading.h
index fd0060d5a..fb5c1c878 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -23,8 +23,7 @@ extern "C" {
 #if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
 
 /**
- * Windows minimalist Pthread Wrapper, based on :
- * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ * Windows minimalist Pthread Wrapper
  */
 #ifdef WINVER
 #  undef WINVER
@@ -62,16 +61,12 @@ extern "C" {
 #define ZSTD_pthread_cond_broadcast(a)  WakeAllConditionVariable((a))
 
 /* ZSTD_pthread_create() and ZSTD_pthread_join() */
-typedef struct {
-    HANDLE handle;
-    void* (*start_routine)(void*);
-    void* arg;
-} ZSTD_pthread_t;
+typedef HANDLE ZSTD_pthread_t;
 
 int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
                    void* (*start_routine) (void*), void* arg);
 
-int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
+int ZSTD_pthread_join(ZSTD_pthread_t thread);
 
 /**
  * add here more wrappers as required
@@ -99,7 +94,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
 
 #define ZSTD_pthread_t                  pthread_t
 #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
-#define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
+#define ZSTD_pthread_join(a)         pthread_join((a),NULL)
 
 #else /* DEBUGLEVEL >= 1 */
 
@@ -124,7 +119,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
 
 #define ZSTD_pthread_t                  pthread_t
 #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
-#define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
+#define ZSTD_pthread_join(a)         pthread_join((a),NULL)
 
 #endif
 
diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c
index d49497cf1..fd237c906 100644
--- a/lib/common/xxhash.c
+++ b/lib/common/xxhash.c
@@ -1,9 +1,9 @@
 /*
  *  xxHash - Fast Hash algorithm
- *  Copyright (c) Yann Collet, Facebook, Inc.
+ *  Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
- *  - xxHash homepage: http://www.xxhash.com
+ *  - xxHash homepage: https://cyan4973.github.io/xxHash/
  *  - xxHash source repository : https://github.com/Cyan4973/xxHash
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/xxhash.h b/lib/common/xxhash.h
index f87102a1d..b8b73290b 100644
--- a/lib/common/xxhash.h
+++ b/lib/common/xxhash.h
@@ -1,9 +1,9 @@
 /*
  *  xxHash - Fast Hash algorithm
- *  Copyright (c) Yann Collet, Facebook, Inc.
+ *  Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
- *  - xxHash homepage: http://www.xxhash.com
+ *  - xxHash homepage: https://cyan4973.github.io/xxHash/
  *  - xxHash source repository : https://github.com/Cyan4973/xxHash
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -1314,7 +1314,7 @@ XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr,
  *   care, as what works on one compiler/platform/optimization level may cause
  *   another to read garbage data or even crash.
  *
- * See http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
+ * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
  *
  * Prefer these methods in priority order (0 > 3 > 1 > 2)
  */
@@ -1655,7 +1655,7 @@ static xxh_u32 XXH_read32(const void* ptr)
 
 /*
  * Portable and safe solution. Generally efficient.
- * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
+ * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
  */
 static xxh_u32 XXH_read32(const void* memPtr)
 {
@@ -2296,7 +2296,7 @@ static xxh_u64 XXH_read64(const void* ptr)
 
 /*
  * Portable and safe solution. Generally efficient.
- * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
+ * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
  */
 static xxh_u64 XXH_read64(const void* memPtr)
 {
diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index 3d7e35b30..320855247 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/zstd_deps.h b/lib/common/zstd_deps.h
index 14211344a..4d767ae9b 100644
--- a/lib/common/zstd_deps.h
+++ b/lib/common/zstd_deps.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index e89226702..37836dc70 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -28,7 +28,6 @@
 #include "../zstd.h"
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "huf.h"
 #ifndef XXH_STATIC_LINKING_ONLY
 #  define XXH_STATIC_LINKING_ONLY  /* XXH64_state_t */
@@ -94,6 +93,7 @@ typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
 
 #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
 #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */)   /* for a non-null block */
+#define MIN_LITERALS_FOR_4_STREAMS 6
 
 typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
 
@@ -113,6 +113,8 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy
 #define LLFSELog    9
 #define OffFSELog   8
 #define MaxFSELog  MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
+#define MaxMLBits 16
+#define MaxLLBits 16
 
 #define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */
 /* Each table cannot take more than #symbols * FSELog bits */
@@ -340,12 +342,13 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore
  *          `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
  */
 typedef struct {
+    size_t nbBlocks;
     size_t compressedSize;
     unsigned long long decompressedBound;
 } ZSTD_frameSizeInfo;   /* decompress & legacy */
 
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);   /* compress & dictBuilder */
-void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
+int ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
 
 /* custom memory allocation functions */
 void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);
diff --git a/lib/common/zstd_trace.h b/lib/common/zstd_trace.h
index 6215f1e70..da20534eb 100644
--- a/lib/common/zstd_trace.h
+++ b/lib/common/zstd_trace.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/clevels.h b/lib/compress/clevels.h
index 7ed2e0049..c18da465f 100644
--- a/lib/compress/clevels.h
+++ b/lib/compress/clevels.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index 21be8c54f..5d3770808 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * FSE : Finite State Entropy encoder
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -91,7 +91,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
     assert(tableLog < 16);   /* required for threshold strategy to work */
 
     /* For explanations on how to distribute symbol values over the table :
-     * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
+     * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
 
      #ifdef __clang_analyzer__
      ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize);   /* useless initialization, just to keep scan-build happy */
@@ -343,16 +343,6 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize,
 *  FSE Compression Code
 ****************************************************************/
 
-FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
-{
-    size_t size;
-    if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
-    size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
-    return (FSE_CTable*)ZSTD_malloc(size);
-}
-
-void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); }
-
 /* provides the minimum logSize to safely represent a distribution */
 static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
 {
@@ -533,40 +523,6 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
     return tableLog;
 }
 
-
-/* fake FSE_CTable, for raw (uncompressed) input */
-size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
-{
-    const unsigned tableSize = 1 << nbBits;
-    const unsigned tableMask = tableSize - 1;
-    const unsigned maxSymbolValue = tableMask;
-    void* const ptr = ct;
-    U16* const tableU16 = ( (U16*) ptr) + 2;
-    void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1);   /* assumption : tableLog >= 1 */
-    FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
-    unsigned s;
-
-    /* Sanity checks */
-    if (nbBits < 1) return ERROR(GENERIC);             /* min size */
-
-    /* header */
-    tableU16[-2] = (U16) nbBits;
-    tableU16[-1] = (U16) maxSymbolValue;
-
-    /* Build table */
-    for (s=0; s not compressible */
-        if (maxCount < (srcSize >> 7)) return 0;   /* Heuristic : not compressible enough */
-    }
-
-    tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
-    CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) );
-
-    /* Write table description header */
-    {   CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
-        op += nc_err;
-    }
-
-    /* Compress */
-    CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) );
-    {   CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) );
-        if (cSize == 0) return 0;   /* not enough space for compressed data */
-        op += cSize;
-    }
-
-    /* check compressibility */
-    if ( (size_t)(op-ostart) >= srcSize-1 ) return 0;
-
-    return op-ostart;
-}
-
-typedef struct {
-    FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
-    union {
-      U32 hist_wksp[HIST_WKSP_SIZE_U32];
-      BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
-    } workspace;
-} fseWkspMax_t;
-
-size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
-{
-    fseWkspMax_t scratchBuffer;
-    DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE));   /* compilation failures here means scratchBuffer is not large enough */
-    if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
-    return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
-}
-
-size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
-{
-    return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
-}
-#endif
-
 #endif   /* FSE_COMMONDEFS_ONLY */
diff --git a/lib/compress/hist.c b/lib/compress/hist.c
index 073c57e75..e2fb431f0 100644
--- a/lib/compress/hist.c
+++ b/lib/compress/hist.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/hist.h b/lib/compress/hist.h
index 228ed48a7..887896b81 100644
--- a/lib/compress/hist.h
+++ b/lib/compress/hist.h
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index e94b398da..29871877a 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -1,6 +1,6 @@
 /* ******************************************************************
  * Huffman encoder, part of New Generation Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -29,7 +29,6 @@
 #include "hist.h"
 #define FSE_STATIC_LINKING_ONLY   /* FSE_optimalTableLog_internal */
 #include "../common/fse.h"        /* header compression */
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "../common/error_private.h"
 #include "../common/bits.h"       /* ZSTD_highbit32 */
@@ -236,6 +235,8 @@ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
     U32 n;
     HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32));
 
+    HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp));
+
     /* check conditions */
     if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC);
     if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
@@ -265,16 +266,6 @@ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
     return ((maxSymbolValue+1)/2) + 1;
 }
 
-/*! HUF_writeCTable() :
-    `CTable` : Huffman tree to save, using huf representation.
-    @return : size of saved CTable */
-size_t HUF_writeCTable (void* dst, size_t maxDstSize,
-                        const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog)
-{
-    HUF_WriteCTableWksp wksp;
-    return HUF_writeCTable_wksp(dst, maxDstSize, CTable, maxSymbolValue, huffLog, &wksp, sizeof(wksp));
-}
-
 
 size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights)
 {
@@ -386,7 +377,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits
 
         /* renorm totalCost from 2^largestBits to 2^targetNbBits
          * note : totalCost is necessarily a multiple of baseCost */
-        assert((totalCost & (baseCost - 1)) == 0);
+        assert(((U32)totalCost & (baseCost - 1)) == 0);
         totalCost >>= (largestBits - targetNbBits);
         assert(totalCost > 0);
 
@@ -486,7 +477,7 @@ typedef struct {
     U16 curr;
 } rankPos;
 
-typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32];
+typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)];
 
 /* Number of buckets available for HUF_sort() */
 #define RANK_POSITION_TABLE_SIZE 192
@@ -505,8 +496,8 @@ typedef struct {
  * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing.
  */
 #define RANK_POSITION_MAX_COUNT_LOG 32
-#define RANK_POSITION_LOG_BUCKETS_BEGIN (RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */
-#define RANK_POSITION_DISTINCT_COUNT_CUTOFF RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */
+#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */)
+#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */)
 
 /* Return the appropriate bucket index for a given count. See definition of
  * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy.
@@ -745,6 +736,8 @@ HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue
     nodeElt* const huffNode = huffNode0+1;
     int nonNullRank;
 
+    HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables));
+
     DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1);
 
     /* safety checks */
@@ -1112,9 +1105,9 @@ HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize,
 static size_t
 HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
                               const void* src, size_t srcSize,
-                              const HUF_CElt* CTable, const int bmi2)
+                              const HUF_CElt* CTable, const int flags)
 {
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
         return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable);
     }
     return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable);
@@ -1125,28 +1118,23 @@ HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
 static size_t
 HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
                               const void* src, size_t srcSize,
-                              const HUF_CElt* CTable, const int bmi2)
+                              const HUF_CElt* CTable, const int flags)
 {
-    (void)bmi2;
+    (void)flags;
     return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
 }
 
 #endif
 
-size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
 {
-    return HUF_compress1X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
-}
-
-size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2)
-{
-    return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2);
+    return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
 }
 
 static size_t
 HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
                               const void* src, size_t srcSize,
-                              const HUF_CElt* CTable, int bmi2)
+                              const HUF_CElt* CTable, int flags)
 {
     size_t const segmentSize = (srcSize+3)/4;   /* first 3 segments */
     const BYTE* ip = (const BYTE*) src;
@@ -1160,7 +1148,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
     op += 6;   /* jumpTable */
 
     assert(op <= oend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         MEM_writeLE16(ostart, (U16)cSize);
         op += cSize;
@@ -1168,7 +1156,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
 
     ip += segmentSize;
     assert(op <= oend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         MEM_writeLE16(ostart+2, (U16)cSize);
         op += cSize;
@@ -1176,7 +1164,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
 
     ip += segmentSize;
     assert(op <= oend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         MEM_writeLE16(ostart+4, (U16)cSize);
         op += cSize;
@@ -1185,7 +1173,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
     ip += segmentSize;
     assert(op <= oend);
     assert(ip <= iend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         op += cSize;
     }
@@ -1193,14 +1181,9 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
     return (size_t)(op-ostart);
 }
 
-size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
 {
-    return HUF_compress4X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
-}
-
-size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2)
-{
-    return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2);
+    return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
 }
 
 typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
@@ -1208,11 +1191,11 @@ typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
 static size_t HUF_compressCTable_internal(
                 BYTE* const ostart, BYTE* op, BYTE* const oend,
                 const void* src, size_t srcSize,
-                HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2)
+                HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags)
 {
     size_t const cSize = (nbStreams==HUF_singleStream) ?
-                         HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2) :
-                         HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2);
+                         HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) :
+                         HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags);
     if (HUF_isError(cSize)) { return cSize; }
     if (cSize==0) { return 0; }   /* uncompressible */
     op += cSize;
@@ -1253,41 +1236,59 @@ unsigned HUF_minTableLog(unsigned symbolCardinality)
     return minBitsSymbols;
 }
 
-unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode)
+unsigned HUF_optimalTableLog(
+            unsigned maxTableLog,
+            size_t srcSize,
+            unsigned maxSymbolValue,
+            void* workSpace, size_t wkspSize,
+            HUF_CElt* table,
+      const unsigned* count,
+            int flags)
 {
-    unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
     assert(srcSize > 1); /* Not supported, RLE should be used instead */
+    assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables));
 
-    if (depthMode == HUF_depth_optimal) { /** Test valid depths and return optimal **/
-        BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
+    if (!(flags & HUF_flags_optimalDepth)) {
+        /* cheap evaluation, based on FSE */
+        return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+    }
+
+    {   BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
         size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp);
-        size_t optSize = ((size_t) ~0);
-        unsigned huffLog;
         size_t maxBits, hSize, newSize;
         const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
+        const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
+        size_t optSize = ((size_t) ~0) - 1;
+        unsigned optLog = maxTableLog, optLogGuess;
 
-        if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog;
+        DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize);
 
-        for (huffLog = HUF_minTableLog(symbolCardinality); huffLog <= maxTableLog; huffLog++) {
-            maxBits = HUF_buildCTable_wksp(table, count,
-                                            maxSymbolValue, huffLog,
-                                            workSpace, wkspSize);
+        /* Search until size increases */
+        for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
+            DEBUGLOG(7, "checking for huffLog=%u", optLogGuess);
+            maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
             if (ERR_isError(maxBits)) continue;
 
-            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits,
-                                              workSpace, wkspSize);
+            if (maxBits < optLogGuess && optLogGuess > minTableLog) break;
+
+            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
+
             if (ERR_isError(hSize)) continue;
 
             newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
 
+            if (newSize > optSize + 1) {
+                break;
+            }
+
             if (newSize < optSize) {
                 optSize = newSize;
-                optLog = huffLog;
+                optLog = optLogGuess;
             }
         }
+        assert(optLog <= HUF_TABLELOG_MAX);
+        return optLog;
     }
-    assert(optLog <= HUF_TABLELOG_MAX);
-    return optLog;
 }
 
 /* HUF_compress_internal() :
@@ -1299,8 +1300,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
                        unsigned maxSymbolValue, unsigned huffLog,
                        HUF_nbStreams_e nbStreams,
                        void* workSpace, size_t wkspSize,
-                       HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
-                 const int bmi2, unsigned suspectUncompressible, HUF_depth_mode depthMode)
+                       HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags)
 {
     HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t));
     BYTE* const ostart = (BYTE*)dst;
@@ -1321,15 +1321,15 @@ HUF_compress_internal (void* dst, size_t dstSize,
     if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
 
     /* Heuristic : If old table is valid, use it for small inputs */
-    if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
+    if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) {
         return HUF_compressCTable_internal(ostart, op, oend,
                                            src, srcSize,
-                                           nbStreams, oldHufTable, bmi2);
+                                           nbStreams, oldHufTable, flags);
     }
 
     /* If uncompressible data is suspected, do a smaller sampling first */
     DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2);
-    if (suspectUncompressible && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) {
+    if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) {
         size_t largestTotal = 0;
         DEBUGLOG(5, "input suspected incompressible : sampling to check");
         {   unsigned maxSymbolValueBegin = maxSymbolValue;
@@ -1357,14 +1357,14 @@ HUF_compress_internal (void* dst, size_t dstSize,
         *repeat = HUF_repeat_none;
     }
     /* Heuristic : use existing table for small inputs */
-    if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
+    if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) {
         return HUF_compressCTable_internal(ostart, op, oend,
                                            src, srcSize,
-                                           nbStreams, oldHufTable, bmi2);
+                                           nbStreams, oldHufTable, flags);
     }
 
     /* Build Huffman Tree */
-    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, depthMode);
+    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags);
     {   size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
                                             maxSymbolValue, huffLog,
                                             &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp));
@@ -1389,7 +1389,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
             if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
                 return HUF_compressCTable_internal(ostart, op, oend,
                                                    src, srcSize,
-                                                   nbStreams, oldHufTable, bmi2);
+                                                   nbStreams, oldHufTable, flags);
         }   }
 
         /* Use the new huffman table */
@@ -1401,48 +1401,20 @@ HUF_compress_internal (void* dst, size_t dstSize,
     }
     return HUF_compressCTable_internal(ostart, op, oend,
                                        src, srcSize,
-                                       nbStreams, table->CTable, bmi2);
-}
-
-
-size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
-                      const void* src, size_t srcSize,
-                      unsigned maxSymbolValue, unsigned huffLog,
-                      void* workSpace, size_t wkspSize)
-{
-    return HUF_compress_internal(dst, dstSize, src, srcSize,
-                                 maxSymbolValue, huffLog, HUF_singleStream,
-                                 workSpace, wkspSize,
-                                 NULL, NULL, 0, 0 /*bmi2*/, 0, HUF_depth_fast);
+                                       nbStreams, table->CTable, flags);
 }
 
 size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
                       const void* src, size_t srcSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize,
-                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat,
-                      int bmi2, unsigned suspectUncompressible, HUF_depth_mode depthMode)
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
 {
     DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize);
     return HUF_compress_internal(dst, dstSize, src, srcSize,
                                  maxSymbolValue, huffLog, HUF_singleStream,
                                  workSpace, wkspSize, hufTable,
-                                 repeat, preferRepeat, bmi2, suspectUncompressible, depthMode);
-}
-
-/* HUF_compress4X_repeat():
- * compress input using 4 streams.
- * provide workspace to generate compression tables */
-size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
-                      const void* src, size_t srcSize,
-                      unsigned maxSymbolValue, unsigned huffLog,
-                      void* workSpace, size_t wkspSize)
-{
-    DEBUGLOG(5, "HUF_compress4X_wksp (srcSize = %zu)", srcSize);
-    return HUF_compress_internal(dst, dstSize, src, srcSize,
-                                 maxSymbolValue, huffLog, HUF_fourStreams,
-                                 workSpace, wkspSize,
-                                 NULL, NULL, 0, 0 /*bmi2*/, 0, HUF_depth_fast);
+                                 repeat, flags);
 }
 
 /* HUF_compress4X_repeat():
@@ -1453,45 +1425,11 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
                       const void* src, size_t srcSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize,
-                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
-                      unsigned suspectUncompressible, HUF_depth_mode depthMode)
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
 {
     DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize);
     return HUF_compress_internal(dst, dstSize, src, srcSize,
                                  maxSymbolValue, huffLog, HUF_fourStreams,
                                  workSpace, wkspSize,
-                                 hufTable, repeat, preferRepeat, bmi2, suspectUncompressible, depthMode);
+                                 hufTable, repeat, flags);
 }
-
-#ifndef ZSTD_NO_UNUSED_FUNCTIONS
-/** HUF_buildCTable() :
- * @return : maxNbBits
- *  Note : count is used before tree is written, so they can safely overlap
- */
-size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits)
-{
-    HUF_buildCTable_wksp_tables workspace;
-    return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, &workspace, sizeof(workspace));
-}
-
-size_t HUF_compress1X (void* dst, size_t dstSize,
-                 const void* src, size_t srcSize,
-                 unsigned maxSymbolValue, unsigned huffLog)
-{
-    U64 workSpace[HUF_WORKSPACE_SIZE_U64];
-    return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
-}
-
-size_t HUF_compress2 (void* dst, size_t dstSize,
-                const void* src, size_t srcSize,
-                unsigned maxSymbolValue, unsigned huffLog)
-{
-    U64 workSpace[HUF_WORKSPACE_SIZE_U64];
-    return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
-}
-
-size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
-    return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT);
-}
-#endif
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 5bc7a9f3f..5b89ca6d2 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,7 +16,6 @@
 #include "hist.h"           /* HIST_countFast_wksp */
 #define FSE_STATIC_LINKING_ONLY   /* FSE_encodeSymbol */
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "zstd_compress_internal.h"
 #include "zstd_compress_sequences.h"
@@ -59,14 +58,17 @@
 *  Helper functions
 ***************************************/
 /* ZSTD_compressBound()
- * Note that the result from this function is only compatible with the "normal"
- * full-block strategy.
- * When there are a lot of small blocks due to frequent flush in streaming mode
- * the overhead of headers can make the compressed data to be larger than the
- * return value of ZSTD_compressBound().
+ * Note that the result from this function is only valid for
+ * the one-pass compression functions.
+ * When employing the streaming mode,
+ * if flushes are frequently altering the size of blocks,
+ * the overhead from block headers can make the compressed data larger
+ * than the return value of ZSTD_compressBound().
  */
 size_t ZSTD_compressBound(size_t srcSize) {
-    return ZSTD_COMPRESSBOUND(srcSize);
+    size_t const r = ZSTD_COMPRESSBOUND(srcSize);
+    if (r==0) return ERROR(srcSize_wrong);
+    return r;
 }
 
 
@@ -275,6 +277,28 @@ static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
     return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable;
 }
 
+static int ZSTD_resolveExternalSequenceValidation(int mode) {
+    return mode;
+}
+
+/* Resolves maxBlockSize to the default if no value is present. */
+static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
+    if (maxBlockSize == 0) {
+        return ZSTD_BLOCKSIZE_MAX;
+    } else {
+        return maxBlockSize;
+    }
+}
+
+static ZSTD_paramSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_paramSwitch_e value, int cLevel) {
+    if (value != ZSTD_ps_auto) return value;
+    if (cLevel < 10) {
+        return ZSTD_ps_disable;
+    } else {
+        return ZSTD_ps_enable;
+    }
+}
+
 /* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
  * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
 static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
@@ -298,6 +322,10 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
     }
     cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
     cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
+    cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
+    cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize);
+    cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes,
+                                                                             cctxParams.compressionLevel);
     assert(!ZSTD_checkCParams(cParams));
     return cctxParams;
 }
@@ -343,10 +371,13 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel)
 #define ZSTD_NO_CLEVEL 0
 
 /**
- * Initializes the cctxParams from params and compressionLevel.
+ * Initializes `cctxParams` from `params` and `compressionLevel`.
  * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL.
  */
-static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_parameters const* params, int compressionLevel)
+static void
+ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams,
+                        const ZSTD_parameters* params,
+                              int compressionLevel)
 {
     assert(!ZSTD_checkCParams(params->cParams));
     ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
@@ -359,6 +390,9 @@ static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_par
     cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, ¶ms->cParams);
     cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, ¶ms->cParams);
     cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->cParams);
+    cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
+    cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize);
+    cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel);
     DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
                 cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
 }
@@ -373,7 +407,7 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete
 
 /**
  * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone.
- * @param param Validated zstd parameters.
+ * @param params Validated zstd parameters.
  */
 static void ZSTD_CCtxParams_setZstdParams(
         ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params)
@@ -581,6 +615,21 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         bounds.upperBound = (int)ZSTD_ps_disable;
         return bounds;
 
+    case ZSTD_c_enableMatchFinderFallback:
+        bounds.lowerBound = 0;
+        bounds.upperBound = 1;
+        return bounds;
+
+    case ZSTD_c_maxBlockSize:
+        bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN;
+        bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
+        return bounds;
+
+    case ZSTD_c_searchForExternalRepcodes:
+        bounds.lowerBound = (int)ZSTD_ps_auto;
+        bounds.upperBound = (int)ZSTD_ps_disable;
+        return bounds;
+
     default:
         bounds.error = ERROR(parameter_unsupported);
         return bounds;
@@ -646,6 +695,9 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_useRowMatchFinder:
     case ZSTD_c_deterministicRefPrefix:
     case ZSTD_c_prefetchCDictTables:
+    case ZSTD_c_enableMatchFinderFallback:
+    case ZSTD_c_maxBlockSize:
+    case ZSTD_c_searchForExternalRepcodes:
     default:
         return 0;
     }
@@ -658,7 +710,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
         if (ZSTD_isUpdateAuthorized(param)) {
             cctx->cParamsChanged = 1;
         } else {
-            RETURN_ERROR(stage_wrong, "can only set params in ctx init stage");
+            RETURN_ERROR(stage_wrong, "can only set params in cctx init stage");
     }   }
 
     switch(param)
@@ -702,6 +754,9 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_useRowMatchFinder:
     case ZSTD_c_deterministicRefPrefix:
     case ZSTD_c_prefetchCDictTables:
+    case ZSTD_c_enableMatchFinderFallback:
+    case ZSTD_c_maxBlockSize:
+    case ZSTD_c_searchForExternalRepcodes:
         break;
 
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@@ -793,14 +848,14 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
 
     case ZSTD_c_forceAttachDict : {
         const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value;
-        BOUNDCHECK(ZSTD_c_forceAttachDict, pref);
+        BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref);
         CCtxParams->attachDictPref = pref;
         return CCtxParams->attachDictPref;
     }
 
     case ZSTD_c_literalCompressionMode : {
         const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value;
-        BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm);
+        BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm);
         CCtxParams->literalCompressionMode = lcm;
         return CCtxParams->literalCompressionMode;
     }
@@ -934,6 +989,22 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value;
         return CCtxParams->prefetchCDictTables;
 
+    case ZSTD_c_enableMatchFinderFallback:
+        BOUNDCHECK(ZSTD_c_enableMatchFinderFallback, value);
+        CCtxParams->enableMatchFinderFallback = value;
+        return CCtxParams->enableMatchFinderFallback;
+
+    case ZSTD_c_maxBlockSize:
+        if (value!=0)    /* 0 ==> default */
+            BOUNDCHECK(ZSTD_c_maxBlockSize, value);
+        CCtxParams->maxBlockSize = value;
+        return CCtxParams->maxBlockSize;
+
+    case ZSTD_c_searchForExternalRepcodes:
+        BOUNDCHECK(ZSTD_c_searchForExternalRepcodes, value);
+        CCtxParams->searchForExternalRepcodes = (ZSTD_paramSwitch_e)value;
+        return CCtxParams->searchForExternalRepcodes;
+
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
 }
@@ -1069,6 +1140,15 @@ size_t ZSTD_CCtxParams_getParameter(
     case ZSTD_c_prefetchCDictTables:
         *value = (int)CCtxParams->prefetchCDictTables;
         break;
+    case ZSTD_c_enableMatchFinderFallback:
+        *value = CCtxParams->enableMatchFinderFallback;
+        break;
+    case ZSTD_c_maxBlockSize:
+        *value = (int)CCtxParams->maxBlockSize;
+        break;
+    case ZSTD_c_searchForExternalRepcodes:
+        *value = (int)CCtxParams->searchForExternalRepcodes;
+        break;
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
     return 0;
@@ -1095,6 +1175,21 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
     return 0;
 }
 
+size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams)
+{
+    DEBUGLOG(4, "ZSTD_CCtx_setCParams");
+    assert(cctx != NULL);
+    if (cctx->streamStage != zcss_init) {
+        /* All parameters in @cparams are allowed to be updated during MT compression.
+         * This must be signaled, so that MT compression picks up the changes */
+        cctx->cParamsChanged = 1;
+    }
+    /* only update if parameters are valid */
+    FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), "");
+    cctx->requestedParams.cParams = cparams;
+    return 0;
+}
+
 size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
     DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize);
@@ -1240,6 +1335,7 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
         RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                         "Can't reset parameters only when not in init stage.");
         ZSTD_clearAllDicts(cctx);
+        ZSTD_memset(&cctx->externalMatchCtx, 0, sizeof(cctx->externalMatchCtx));
         return ZSTD_CCtxParams_reset(&cctx->requestedParams);
     }
     return 0;
@@ -1336,7 +1432,8 @@ static ZSTD_compressionParameters
 ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
                             unsigned long long srcSize,
                             size_t dictSize,
-                            ZSTD_cParamMode_e mode)
+                            ZSTD_cParamMode_e mode,
+                            ZSTD_paramSwitch_e useRowMatchFinder)
 {
     const U64 minSrcSize = 513; /* (1<<9) + 1 */
     const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
@@ -1370,8 +1467,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
     }
 
     /* resize windowLog if input is small enough, to use less memory */
-    if ( (srcSize < maxWindowResize)
-      && (dictSize < maxWindowResize) )  {
+    if ( (srcSize <= maxWindowResize)
+      && (dictSize <= maxWindowResize) )  {
         U32 const tSize = (U32)(srcSize + dictSize);
         static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
         U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :
@@ -1389,11 +1486,40 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
     if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
         cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* minimum wlog required for valid frame header */
 
+    /* We can't use more than 32 bits of hash in total, so that means that we require:
+     * (hashLog + 8) <= 32 && (chainLog + 8) <= 32
+     */
     if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) {
         U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS;
         if (cPar.hashLog > maxShortCacheHashLog) {
             cPar.hashLog = maxShortCacheHashLog;
         }
+        if (cPar.chainLog > maxShortCacheHashLog) {
+            cPar.chainLog = maxShortCacheHashLog;
+        }
+    }
+
+
+    /* At this point, we aren't 100% sure if we are using the row match finder.
+     * Unless it is explicitly disabled, conservatively assume that it is enabled.
+     * In this case it will only be disabled for small sources, so shrinking the
+     * hash log a little bit shouldn't result in any ratio loss.
+     */
+    if (useRowMatchFinder == ZSTD_ps_auto)
+        useRowMatchFinder = ZSTD_ps_enable;
+
+    /* We can't hash more than 32-bits in total. So that means that we require:
+     * (hashLog - rowLog + 8) <= 32
+     */
+    if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) {
+        /* Switch to 32-entry rows if searchLog is 5 (or more) */
+        U32 const rowLog = BOUNDED(4, cPar.searchLog, 6);
+        U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS;
+        U32 const maxHashLog = maxRowHashLog + rowLog;
+        assert(cPar.hashLog >= rowLog);
+        if (cPar.hashLog > maxHashLog) {
+            cPar.hashLog = maxHashLog;
+        }
     }
 
     return cPar;
@@ -1406,7 +1532,7 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar,
 {
     cPar = ZSTD_clampCParams(cPar);   /* resulting cPar is necessarily valid (all parameters within range) */
     if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN;
-    return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown);
+    return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto);
 }
 
 static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
@@ -1437,7 +1563,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
     ZSTD_overrideCParams(&cParams, &CCtxParams->cParams);
     assert(!ZSTD_checkCParams(cParams));
     /* srcSizeHint == 0 means 0 */
-    return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode);
+    return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder);
 }
 
 static size_t
@@ -1482,6 +1608,13 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
     return tableSpace + optSpace + slackSpace + lazyAdditionalSpace;
 }
 
+/* Helper function for calculating memory requirements.
+ * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */
+static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useExternalMatchFinder) {
+    U32 const divider = (minMatch==3 || useExternalMatchFinder) ? 3 : 4;
+    return blockSize / divider;
+}
+
 static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const ZSTD_compressionParameters* cParams,
         const ldmParams_t* ldmParams,
@@ -1489,12 +1622,13 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const ZSTD_paramSwitch_e useRowMatchFinder,
         const size_t buffInSize,
         const size_t buffOutSize,
-        const U64 pledgedSrcSize)
+        const U64 pledgedSrcSize,
+        int useExternalMatchFinder,
+        size_t maxBlockSize)
 {
     size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
-    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
-    U32    const divider = (cParams->minMatch==3) ? 3 : 4;
-    size_t const maxNbSeq = blockSize / divider;
+    size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
+    size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder);
     size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
                             + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
                             + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
@@ -1513,6 +1647,11 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
 
     size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
 
+    size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+    size_t const externalSeqSpace = useExternalMatchFinder
+        ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
+        : 0;
+
     size_t const neededSpace =
         cctxSpace +
         entropySpace +
@@ -1521,7 +1660,8 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         ldmSeqSpace +
         matchStateSize +
         tokenSpace +
-        bufferSpace;
+        bufferSpace +
+        externalSeqSpace;
 
     DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
     return neededSpace;
@@ -1539,7 +1679,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
      * be needed. However, we still allocate two 0-sized buffers, which can
      * take space under ASAN. */
     return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
-        &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN);
+        &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
 }
 
 size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
@@ -1589,7 +1729,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
     RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
     {   ZSTD_compressionParameters const cParams =
                 ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
-        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
+        size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog);
         size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered)
                 ? ((size_t)1 << cParams.windowLog) + blockSize
                 : 0;
@@ -1600,7 +1740,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 
         return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
             &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
-            ZSTD_CONTENTSIZE_UNKNOWN);
+            ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
     }
 }
 
@@ -1874,6 +2014,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     assert(params->useRowMatchFinder != ZSTD_ps_auto);
     assert(params->useBlockSplitter != ZSTD_ps_auto);
     assert(params->ldmParams.enableLdm != ZSTD_ps_auto);
+    assert(params->maxBlockSize != 0);
     if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
         /* Adjust long distance matching parameters */
         ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams);
@@ -1882,9 +2023,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     }
 
     {   size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
-        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
-        U32    const divider = (params->cParams.minMatch==3) ? 3 : 4;
-        size_t const maxNbSeq = blockSize / divider;
+        size_t const blockSize = MIN(params->maxBlockSize, windowSize);
+        size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useExternalMatchFinder);
         size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
                 ? ZSTD_compressBound(blockSize) + 1
                 : 0;
@@ -1901,7 +2041,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         size_t const neededSpace =
             ZSTD_estimateCCtxSize_usingCCtxParams_internal(
                 ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
-                buffInSize, buffOutSize, pledgedSrcSize);
+                buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder, params->maxBlockSize);
         int resizeWorkspace;
 
         FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
@@ -2014,6 +2154,14 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
             zc->ldmState.loadedDictEnd = 0;
         }
 
+        /* reserve space for block-level external sequences */
+        if (params->useExternalMatchFinder) {
+            size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+            zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq;
+            zc->externalMatchCtx.seqBuffer =
+                (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
+        }
+
         DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
         assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace));
 
@@ -2087,7 +2235,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
         }
 
         params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize,
-                                                     cdict->dictContentSize, ZSTD_cpm_attachDict);
+                                                     cdict->dictContentSize, ZSTD_cpm_attachDict,
+                                                     params.useRowMatchFinder);
         params.cParams.windowLog = windowLog;
         params.useRowMatchFinder = cdict->useRowMatchFinder;    /* cdict overrides */
         FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize,
@@ -2271,6 +2420,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
         params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter;
         params.ldmParams = srcCCtx->appliedParams.ldmParams;
         params.fParams = fParams;
+        params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize;
         ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize,
                                 /* loadedDictSize */ 0,
                                 ZSTDcrp_leaveDirty, zbuff);
@@ -2430,7 +2580,7 @@ static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* par
 
 /* See doc/zstd_compression_format.md for detailed format description */
 
-void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
+int ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
 {
     const seqDef* const sequences = seqStorePtr->sequencesStart;
     BYTE* const llCodeTable = seqStorePtr->llCode;
@@ -2438,18 +2588,24 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
     BYTE* const mlCodeTable = seqStorePtr->mlCode;
     U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     U32 u;
+    int longOffsets = 0;
     assert(nbSeq <= seqStorePtr->maxNbSeq);
     for (u=0; u= STREAM_ACCUMULATOR_MIN));
+        if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN)
+            longOffsets = 1;
     }
     if (seqStorePtr->longLengthType==ZSTD_llt_literalLength)
         llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
     if (seqStorePtr->longLengthType==ZSTD_llt_matchLength)
         mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
+    return longOffsets;
 }
 
 /* ZSTD_useTargetCBlockSize():
@@ -2483,6 +2639,7 @@ typedef struct {
     U32 MLtype;
     size_t size;
     size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
+    int longOffsets;
 } ZSTD_symbolEncodingTypeStats_t;
 
 /* ZSTD_buildSequencesStatistics():
@@ -2493,11 +2650,13 @@ typedef struct {
  * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32)
  */
 static ZSTD_symbolEncodingTypeStats_t
-ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq,
-                        const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
-                              BYTE* dst, const BYTE* const dstEnd,
-                              ZSTD_strategy strategy, unsigned* countWorkspace,
-                              void* entropyWorkspace, size_t entropyWkspSize) {
+ZSTD_buildSequencesStatistics(
+                const seqStore_t* seqStorePtr, size_t nbSeq,
+                const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
+                      BYTE* dst, const BYTE* const dstEnd,
+                      ZSTD_strategy strategy, unsigned* countWorkspace,
+                      void* entropyWorkspace, size_t entropyWkspSize)
+{
     BYTE* const ostart = dst;
     const BYTE* const oend = dstEnd;
     BYTE* op = ostart;
@@ -2511,7 +2670,7 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq,
 
     stats.lastCountSize = 0;
     /* convert length/distances into codes */
-    ZSTD_seqToCodes(seqStorePtr);
+    stats.longOffsets = ZSTD_seqToCodes(seqStorePtr);
     assert(op <= oend);
     assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */
     /* build CTable for Literal Lengths */
@@ -2616,22 +2775,22 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq,
  */
 #define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20
 MEM_STATIC size_t
-ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
-                          const ZSTD_entropyCTables_t* prevEntropy,
-                                ZSTD_entropyCTables_t* nextEntropy,
-                          const ZSTD_CCtx_params* cctxParams,
-                                void* dst, size_t dstCapacity,
-                                void* entropyWorkspace, size_t entropyWkspSize,
-                          const int bmi2)
+ZSTD_entropyCompressSeqStore_internal(
+                        const seqStore_t* seqStorePtr,
+                        const ZSTD_entropyCTables_t* prevEntropy,
+                              ZSTD_entropyCTables_t* nextEntropy,
+                        const ZSTD_CCtx_params* cctxParams,
+                              void* dst, size_t dstCapacity,
+                              void* entropyWorkspace, size_t entropyWkspSize,
+                        const int bmi2)
 {
-    const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
     ZSTD_strategy const strategy = cctxParams->cParams.strategy;
     unsigned* count = (unsigned*)entropyWorkspace;
     FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
     FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
     FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
     const seqDef* const sequences = seqStorePtr->sequencesStart;
-    const size_t nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
+    const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     const BYTE* const ofCodeTable = seqStorePtr->ofCode;
     const BYTE* const llCodeTable = seqStorePtr->llCode;
     const BYTE* const mlCodeTable = seqStorePtr->mlCode;
@@ -2639,6 +2798,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
     BYTE* const oend = ostart + dstCapacity;
     BYTE* op = ostart;
     size_t lastCountSize;
+    int longOffsets = 0;
 
     entropyWorkspace = count + (MaxSeq + 1);
     entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
@@ -2649,21 +2809,20 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
 
     /* Compress literals */
     {   const BYTE* const literals = seqStorePtr->litStart;
-        size_t const numSequences = seqStorePtr->sequences - seqStorePtr->sequencesStart;
-        size_t const numLiterals = seqStorePtr->lit - seqStorePtr->litStart;
+        size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+        size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
         /* Base suspicion of uncompressibility on ratio of literals to sequences */
         unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
         size_t const litSize = (size_t)(seqStorePtr->lit - literals);
 
-        HUF_depth_mode depthMode = cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast;
         size_t const cSize = ZSTD_compressLiterals(
-                                    &prevEntropy->huf, &nextEntropy->huf,
-                                    cctxParams->cParams.strategy,
-                                    ZSTD_literalsCompressionIsDisabled(cctxParams),
                                     op, dstCapacity,
                                     literals, litSize,
                                     entropyWorkspace, entropyWkspSize,
-                                    bmi2, suspectUncompressible, depthMode);
+                                    &prevEntropy->huf, &nextEntropy->huf,
+                                    cctxParams->cParams.strategy,
+                                    ZSTD_literalsCompressionIsDisabled(cctxParams),
+                                    suspectUncompressible, bmi2);
         FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed");
         assert(cSize <= dstCapacity);
         op += cSize;
@@ -2701,6 +2860,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
         *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2));
         lastCountSize = stats.lastCountSize;
         op += stats.size;
+        longOffsets = stats.longOffsets;
     }
 
     {   size_t const bitstreamSize = ZSTD_encodeSequences(
@@ -2735,14 +2895,15 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
 }
 
 MEM_STATIC size_t
-ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr,
-                       const ZSTD_entropyCTables_t* prevEntropy,
-                             ZSTD_entropyCTables_t* nextEntropy,
-                       const ZSTD_CCtx_params* cctxParams,
-                             void* dst, size_t dstCapacity,
-                             size_t srcSize,
-                             void* entropyWorkspace, size_t entropyWkspSize,
-                             int bmi2)
+ZSTD_entropyCompressSeqStore(
+                    const seqStore_t* seqStorePtr,
+                    const ZSTD_entropyCTables_t* prevEntropy,
+                          ZSTD_entropyCTables_t* nextEntropy,
+                    const ZSTD_CCtx_params* cctxParams,
+                          void* dst, size_t dstCapacity,
+                          size_t srcSize,
+                          void* entropyWorkspace, size_t entropyWkspSize,
+                          int bmi2)
 {
     size_t const cSize = ZSTD_entropyCompressSeqStore_internal(
                             seqStorePtr, prevEntropy, nextEntropy, cctxParams,
@@ -2763,6 +2924,10 @@ ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr,
         if (cSize >= maxCSize) return 0;  /* block not compressed */
     }
     DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
+    /* libzstd decoder before  > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly.
+     * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above.
+     */
+    assert(cSize < ZSTD_BLOCKSIZE_MAX);
     return cSize;
 }
 
@@ -2857,6 +3022,72 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr)
     ssPtr->longLengthType = ZSTD_llt_none;
 }
 
+/* ZSTD_postProcessExternalMatchFinderResult() :
+ * Validates and post-processes sequences obtained through the external matchfinder API:
+ *   - Checks whether nbExternalSeqs represents an error condition.
+ *   - Appends a block delimiter to outSeqs if one is not already present.
+ *     See zstd.h for context regarding block delimiters.
+ * Returns the number of sequences after post-processing, or an error code. */
+static size_t ZSTD_postProcessExternalMatchFinderResult(
+    ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize
+) {
+    RETURN_ERROR_IF(
+        nbExternalSeqs > outSeqsCapacity,
+        externalMatchFinder_failed,
+        "External matchfinder returned error code %lu",
+        (unsigned long)nbExternalSeqs
+    );
+
+    RETURN_ERROR_IF(
+        nbExternalSeqs == 0 && srcSize > 0,
+        externalMatchFinder_failed,
+        "External matchfinder produced zero sequences for a non-empty src buffer!"
+    );
+
+    if (srcSize == 0) {
+        ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence));
+        return 1;
+    }
+
+    {
+        ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1];
+
+        /* We can return early if lastSeq is already a block delimiter. */
+        if (lastSeq.offset == 0 && lastSeq.matchLength == 0) {
+            return nbExternalSeqs;
+        }
+
+        /* This error condition is only possible if the external matchfinder
+         * produced an invalid parse, by definition of ZSTD_sequenceBound(). */
+        RETURN_ERROR_IF(
+            nbExternalSeqs == outSeqsCapacity,
+            externalMatchFinder_failed,
+            "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!"
+        );
+
+        /* lastSeq is not a block delimiter, so we need to append one. */
+        ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence));
+        return nbExternalSeqs + 1;
+    }
+}
+
+/* ZSTD_fastSequenceLengthSum() :
+ * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*.
+ * Similar to another function in zstd_compress.c (determine_blockSize),
+ * except it doesn't check for a block delimiter to end summation.
+ * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P).
+ * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */
+static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) {
+    size_t matchLenSum, litLenSum, i;
+    matchLenSum = 0;
+    litLenSum = 0;
+    for (i = 0; i < seqBufSize; i++) {
+        litLenSum += seqBuf[i].litLength;
+        matchLenSum += seqBuf[i].matchLength;
+    }
+    return litLenSum + matchLenSum;
+}
+
 typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;
 
 static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
@@ -2904,6 +3135,15 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
         }
         if (zc->externSeqStore.pos < zc->externSeqStore.size) {
             assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable);
+
+            /* External matchfinder + LDM is technically possible, just not implemented yet.
+             * We need to revisit soon and implement it. */
+            RETURN_ERROR_IF(
+                zc->appliedParams.useExternalMatchFinder,
+                parameter_combination_unsupported,
+                "Long-distance matching with external matchfinder enabled is not currently supported."
+            );
+
             /* Updates ldmSeqStore.pos */
             lastLLSize =
                 ZSTD_ldm_blockCompress(&zc->externSeqStore,
@@ -2915,6 +3155,14 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
         } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) {
             rawSeqStore_t ldmSeqStore = kNullRawSeqStore;
 
+            /* External matchfinder + LDM is technically possible, just not implemented yet.
+             * We need to revisit soon and implement it. */
+            RETURN_ERROR_IF(
+                zc->appliedParams.useExternalMatchFinder,
+                parameter_combination_unsupported,
+                "Long-distance matching with external matchfinder enabled is not currently supported."
+            );
+
             ldmSeqStore.seq = zc->ldmSequences;
             ldmSeqStore.capacity = zc->maxNbLdmSequences;
             /* Updates ldmSeqStore.size */
@@ -2929,7 +3177,68 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
                                        zc->appliedParams.useRowMatchFinder,
                                        src, srcSize);
             assert(ldmSeqStore.pos == ldmSeqStore.size);
-        } else {   /* not long range mode */
+        } else if (zc->appliedParams.useExternalMatchFinder) {
+            assert(
+                zc->externalMatchCtx.seqBufferCapacity >= ZSTD_sequenceBound(srcSize)
+            );
+            assert(zc->externalMatchCtx.mFinder != NULL);
+
+            {   U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog;
+
+                size_t const nbExternalSeqs = (zc->externalMatchCtx.mFinder)(
+                    zc->externalMatchCtx.mState,
+                    zc->externalMatchCtx.seqBuffer,
+                    zc->externalMatchCtx.seqBufferCapacity,
+                    src, srcSize,
+                    NULL, 0,  /* dict and dictSize, currently not supported */
+                    zc->appliedParams.compressionLevel,
+                    windowSize
+                );
+
+                size_t const nbPostProcessedSeqs = ZSTD_postProcessExternalMatchFinderResult(
+                    zc->externalMatchCtx.seqBuffer,
+                    nbExternalSeqs,
+                    zc->externalMatchCtx.seqBufferCapacity,
+                    srcSize
+                );
+
+                /* Return early if there is no error, since we don't need to worry about last literals */
+                if (!ZSTD_isError(nbPostProcessedSeqs)) {
+                    ZSTD_sequencePosition seqPos = {0,0,0};
+                    size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs);
+                    RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!");
+                    FORWARD_IF_ERROR(
+                        ZSTD_copySequencesToSeqStoreExplicitBlockDelim(
+                            zc, &seqPos,
+                            zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs,
+                            src, srcSize,
+                            zc->appliedParams.searchForExternalRepcodes
+                        ),
+                        "Failed to copy external sequences to seqStore!"
+                    );
+                    ms->ldmSeqStore = NULL;
+                    DEBUGLOG(5, "Copied %lu sequences from external matchfinder to internal seqStore.", (unsigned long)nbExternalSeqs);
+                    return ZSTDbss_compress;
+                }
+
+                /* Propagate the error if fallback is disabled */
+                if (!zc->appliedParams.enableMatchFinderFallback) {
+                    return nbPostProcessedSeqs;
+                }
+
+                /* Fallback to software matchfinder */
+                {   ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
+                                                                                            zc->appliedParams.useRowMatchFinder,
+                                                                                            dictMode);
+                    ms->ldmSeqStore = NULL;
+                    DEBUGLOG(
+                        5,
+                        "External matchfinder returned error code %lu. Falling back to internal matchfinder.",
+                        (unsigned long)nbExternalSeqs
+                    );
+                    lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
+            }   }
+        } else {   /* not long range mode and no external matchfinder */
             ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
                                                                                     zc->appliedParams.useRowMatchFinder,
                                                                                     dictMode);
@@ -3055,19 +3364,17 @@ static int ZSTD_isRLE(const BYTE* src, size_t length) {
     const size_t unrollMask = unrollSize - 1;
     const size_t prefixLength = length & unrollMask;
     size_t i;
-    size_t u;
     if (length == 1) return 1;
     /* Check if prefix is RLE first before using unrolled loop */
     if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) {
         return 0;
     }
     for (i = prefixLength; i != length; i += unrollSize) {
+        size_t u;
         for (u = 0; u < unrollSize; u += sizeof(size_t)) {
             if (MEM_readST(ip + i + u) != valueST) {
                 return 0;
-            }
-        }
-    }
+    }   }   }
     return 1;
 }
 
@@ -3083,7 +3390,8 @@ static int ZSTD_maybeRLE(seqStore_t const* seqStore)
     return nbSeqs < 4 && nbLits < 10;
 }
 
-static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
+static void
+ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
 {
     ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock;
     bs->prevCBlock = bs->nextCBlock;
@@ -3091,7 +3399,9 @@ static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* c
 }
 
 /* Writes the block header */
-static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) {
+static void
+writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock)
+{
     U32 const cBlockHeader = cSize == 1 ?
                         lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
                         lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
@@ -3104,13 +3414,16 @@ static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastB
  *  Stores literals block type (raw, rle, compressed, repeat) and
  *  huffman description table to hufMetadata.
  *  Requires ENTROPY_WORKSPACE_SIZE workspace
- *  @return : size of huffman description table or error code */
-static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
-                                            const ZSTD_hufCTables_t* prevHuf,
-                                                  ZSTD_hufCTables_t* nextHuf,
-                                                  ZSTD_hufCTablesMetadata_t* hufMetadata,
-                                                  const int literalsCompressionIsDisabled,
-                                                  void* workspace, size_t wkspSize, HUF_depth_mode depthMode)
+ * @return : size of huffman description table, or an error code
+ */
+static size_t
+ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
+                               const ZSTD_hufCTables_t* prevHuf,
+                                     ZSTD_hufCTables_t* nextHuf,
+                                     ZSTD_hufCTablesMetadata_t* hufMetadata,
+                               const int literalsCompressionIsDisabled,
+                                     void* workspace, size_t wkspSize,
+                                     int hufFlags)
 {
     BYTE* const wkspStart = (BYTE*)workspace;
     BYTE* const wkspEnd = wkspStart + wkspSize;
@@ -3135,74 +3448,77 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
 
     /* small ? don't even attempt compression (speed opt) */
 #ifndef COMPRESS_LITERALS_SIZE_MIN
-#define COMPRESS_LITERALS_SIZE_MIN 63
+# define COMPRESS_LITERALS_SIZE_MIN 63  /* heuristic */
 #endif
     {   size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
         if (srcSize <= minLitSize) {
             DEBUGLOG(5, "set_basic - too small");
             hufMetadata->hType = set_basic;
             return 0;
-        }
-    }
+    }   }
 
     /* Scan input and build symbol stats */
-    {   size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize);
+    {   size_t const largest =
+            HIST_count_wksp (countWksp, &maxSymbolValue,
+                            (const BYTE*)src, srcSize,
+                            workspace, wkspSize);
         FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
         if (largest == srcSize) {
+            /* only one literal symbol */
             DEBUGLOG(5, "set_rle");
             hufMetadata->hType = set_rle;
             return 0;
         }
         if (largest <= (srcSize >> 7)+4) {
+            /* heuristic: likely not compressible */
             DEBUGLOG(5, "set_basic - no gain");
             hufMetadata->hType = set_basic;
             return 0;
-        }
-    }
+    }   }
 
     /* Validate the previous Huffman table */
-    if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
+    if (repeat == HUF_repeat_check
+      && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
         repeat = HUF_repeat_none;
     }
 
     /* Build Huffman Tree */
     ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
-    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, depthMode);
+    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags);
     assert(huffLog <= LitHufLog);
     {   size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
                                                     maxSymbolValue, huffLog,
                                                     nodeWksp, nodeWkspSize);
         FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
         huffLog = (U32)maxBits;
-        {   /* Build and write the CTable */
-            size_t const newCSize = HUF_estimateCompressedSize(
-                    (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
-            size_t const hSize = HUF_writeCTable_wksp(
-                    hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
-                    (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
-                    nodeWksp, nodeWkspSize);
-            /* Check against repeating the previous CTable */
-            if (repeat != HUF_repeat_none) {
-                size_t const oldCSize = HUF_estimateCompressedSize(
-                        (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
-                if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
-                    DEBUGLOG(5, "set_repeat - smaller");
-                    ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-                    hufMetadata->hType = set_repeat;
-                    return 0;
-                }
-            }
-            if (newCSize + hSize >= srcSize) {
-                DEBUGLOG(5, "set_basic - no gains");
+    }
+    {   /* Build and write the CTable */
+        size_t const newCSize = HUF_estimateCompressedSize(
+                (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
+        size_t const hSize = HUF_writeCTable_wksp(
+                hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
+                (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
+                nodeWksp, nodeWkspSize);
+        /* Check against repeating the previous CTable */
+        if (repeat != HUF_repeat_none) {
+            size_t const oldCSize = HUF_estimateCompressedSize(
+                    (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
+            if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
+                DEBUGLOG(5, "set_repeat - smaller");
                 ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-                hufMetadata->hType = set_basic;
+                hufMetadata->hType = set_repeat;
                 return 0;
-            }
-            DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
-            hufMetadata->hType = set_compressed;
-            nextHuf->repeatMode = HUF_repeat_check;
-            return hSize;
+        }   }
+        if (newCSize + hSize >= srcSize) {
+            DEBUGLOG(5, "set_basic - no gains");
+            ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+            hufMetadata->hType = set_basic;
+            return 0;
         }
+        DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
+        hufMetadata->hType = set_compressed;
+        nextHuf->repeatMode = HUF_repeat_check;
+        return hSize;
     }
 }
 
@@ -3212,8 +3528,9 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
  * and updates nextEntropy to the appropriate repeatMode.
  */
 static ZSTD_symbolEncodingTypeStats_t
-ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) {
-    ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0};
+ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy)
+{
+    ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0};
     nextEntropy->litlength_repeatMode = FSE_repeat_none;
     nextEntropy->offcode_repeatMode = FSE_repeat_none;
     nextEntropy->matchlength_repeatMode = FSE_repeat_none;
@@ -3224,16 +3541,18 @@ ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) {
  *  Builds entropy for the sequences.
  *  Stores symbol compression modes and fse table to fseMetadata.
  *  Requires ENTROPY_WORKSPACE_SIZE wksp.
- *  @return : size of fse tables or error code */
-static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr,
-                                              const ZSTD_fseCTables_t* prevEntropy,
-                                                    ZSTD_fseCTables_t* nextEntropy,
-                                              const ZSTD_CCtx_params* cctxParams,
-                                                    ZSTD_fseCTablesMetadata_t* fseMetadata,
-                                                    void* workspace, size_t wkspSize)
+ * @return : size of fse tables or error code */
+static size_t
+ZSTD_buildBlockEntropyStats_sequences(
+                const seqStore_t* seqStorePtr,
+                const ZSTD_fseCTables_t* prevEntropy,
+                      ZSTD_fseCTables_t* nextEntropy,
+                const ZSTD_CCtx_params* cctxParams,
+                      ZSTD_fseCTablesMetadata_t* fseMetadata,
+                      void* workspace, size_t wkspSize)
 {
     ZSTD_strategy const strategy = cctxParams->cParams.strategy;
-    size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
+    size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     BYTE* const ostart = fseMetadata->fseTablesBuffer;
     BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
     BYTE* op = ostart;
@@ -3260,25 +3579,27 @@ static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr,
 /** ZSTD_buildBlockEntropyStats() :
  *  Builds entropy for the block.
  *  Requires workspace size ENTROPY_WORKSPACE_SIZE
- *
- *  @return : 0 on success or error code
+ * @return : 0 on success, or an error code
+ *  Note : also employed in superblock
  */
-size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
-                             const ZSTD_entropyCTables_t* prevEntropy,
-                                   ZSTD_entropyCTables_t* nextEntropy,
-                             const ZSTD_CCtx_params* cctxParams,
-                                   ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                   void* workspace, size_t wkspSize)
+size_t ZSTD_buildBlockEntropyStats(
+            const seqStore_t* seqStorePtr,
+            const ZSTD_entropyCTables_t* prevEntropy,
+                  ZSTD_entropyCTables_t* nextEntropy,
+            const ZSTD_CCtx_params* cctxParams,
+                  ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                  void* workspace, size_t wkspSize)
 {
-    size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
-    HUF_depth_mode depthMode = cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast;
+    size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
+    int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD);
+    int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0;
 
     entropyMetadata->hufMetadata.hufDesSize =
         ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
                                             &prevEntropy->huf, &nextEntropy->huf,
                                             &entropyMetadata->hufMetadata,
                                             ZSTD_literalsCompressionIsDisabled(cctxParams),
-                                            workspace, wkspSize, depthMode);
+                                            workspace, wkspSize, hufFlags);
 
     FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
     entropyMetadata->fseMetadata.fseTablesSize =
@@ -3292,11 +3613,12 @@ size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
 }
 
 /* Returns the size estimate for the literals section (header + content) of a block */
-static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
-                                                const ZSTD_hufCTables_t* huf,
-                                                const ZSTD_hufCTablesMetadata_t* hufMetadata,
-                                                void* workspace, size_t wkspSize,
-                                                int writeEntropy)
+static size_t
+ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
+                               const ZSTD_hufCTables_t* huf,
+                               const ZSTD_hufCTablesMetadata_t* hufMetadata,
+                               void* workspace, size_t wkspSize,
+                               int writeEntropy)
 {
     unsigned* const countWksp = (unsigned*)workspace;
     unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
@@ -3318,12 +3640,13 @@ static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSiz
 }
 
 /* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */
-static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
-                        const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
-                        const FSE_CTable* fseCTable,
-                        const U8* additionalBits,
-                        short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
-                        void* workspace, size_t wkspSize)
+static size_t
+ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
+                    const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
+                    const FSE_CTable* fseCTable,
+                    const U8* additionalBits,
+                    short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+                    void* workspace, size_t wkspSize)
 {
     unsigned* const countWksp = (unsigned*)workspace;
     const BYTE* ctp = codeTable;
@@ -3355,99 +3678,107 @@ static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
 }
 
 /* Returns the size estimate for the sequences section (header + content) of a block */
-static size_t ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable,
-                                                  const BYTE* llCodeTable,
-                                                  const BYTE* mlCodeTable,
-                                                  size_t nbSeq,
-                                                  const ZSTD_fseCTables_t* fseTables,
-                                                  const ZSTD_fseCTablesMetadata_t* fseMetadata,
-                                                  void* workspace, size_t wkspSize,
-                                                  int writeEntropy)
+static size_t
+ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable,
+                                 const BYTE* llCodeTable,
+                                 const BYTE* mlCodeTable,
+                                 size_t nbSeq,
+                                 const ZSTD_fseCTables_t* fseTables,
+                                 const ZSTD_fseCTablesMetadata_t* fseMetadata,
+                                 void* workspace, size_t wkspSize,
+                                 int writeEntropy)
 {
     size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ);
     size_t cSeqSizeEstimate = 0;
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff,
-                                         fseTables->offcodeCTable, NULL,
-                                         OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
-                                         workspace, wkspSize);
+                                    fseTables->offcodeCTable, NULL,
+                                    OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
+                                    workspace, wkspSize);
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL,
-                                         fseTables->litlengthCTable, LL_bits,
-                                         LL_defaultNorm, LL_defaultNormLog, MaxLL,
-                                         workspace, wkspSize);
+                                    fseTables->litlengthCTable, LL_bits,
+                                    LL_defaultNorm, LL_defaultNormLog, MaxLL,
+                                    workspace, wkspSize);
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML,
-                                         fseTables->matchlengthCTable, ML_bits,
-                                         ML_defaultNorm, ML_defaultNormLog, MaxML,
-                                         workspace, wkspSize);
+                                    fseTables->matchlengthCTable, ML_bits,
+                                    ML_defaultNorm, ML_defaultNormLog, MaxML,
+                                    workspace, wkspSize);
     if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
     return cSeqSizeEstimate + sequencesSectionHeaderSize;
 }
 
 /* Returns the size estimate for a given stream of literals, of, ll, ml */
-static size_t ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
-                                     const BYTE* ofCodeTable,
-                                     const BYTE* llCodeTable,
-                                     const BYTE* mlCodeTable,
-                                     size_t nbSeq,
-                                     const ZSTD_entropyCTables_t* entropy,
-                                     const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                     void* workspace, size_t wkspSize,
-                                     int writeLitEntropy, int writeSeqEntropy) {
+static size_t
+ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
+                       const BYTE* ofCodeTable,
+                       const BYTE* llCodeTable,
+                       const BYTE* mlCodeTable,
+                       size_t nbSeq,
+                       const ZSTD_entropyCTables_t* entropy,
+                       const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                       void* workspace, size_t wkspSize,
+                       int writeLitEntropy, int writeSeqEntropy)
+{
     size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize,
-                                                         &entropy->huf, &entropyMetadata->hufMetadata,
-                                                         workspace, wkspSize, writeLitEntropy);
+                                    &entropy->huf, &entropyMetadata->hufMetadata,
+                                    workspace, wkspSize, writeLitEntropy);
     size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
-                                                         nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
-                                                         workspace, wkspSize, writeSeqEntropy);
+                                    nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
+                                    workspace, wkspSize, writeSeqEntropy);
     return seqSize + literalsSize + ZSTD_blockHeaderSize;
 }
 
 /* Builds entropy statistics and uses them for blocksize estimation.
  *
- * Returns the estimated compressed size of the seqStore, or a zstd error.
+ * @return: estimated compressed size of the seqStore, or a zstd error.
  */
-static size_t ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc) {
-    ZSTD_entropyCTablesMetadata_t* entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
+static size_t
+ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc)
+{
+    ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
     DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()");
     FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore,
                     &zc->blockState.prevCBlock->entropy,
                     &zc->blockState.nextCBlock->entropy,
                     &zc->appliedParams,
                     entropyMetadata,
-                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
-    return ZSTD_estimateBlockSize(seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
+                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), "");
+    return ZSTD_estimateBlockSize(
+                    seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
                     seqStore->ofCode, seqStore->llCode, seqStore->mlCode,
                     (size_t)(seqStore->sequences - seqStore->sequencesStart),
-                    &zc->blockState.nextCBlock->entropy, entropyMetadata, zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
+                    &zc->blockState.nextCBlock->entropy,
+                    entropyMetadata,
+                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
                     (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1);
 }
 
 /* Returns literals bytes represented in a seqStore */
-static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) {
+static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore)
+{
     size_t literalsBytes = 0;
-    size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart;
+    size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
     size_t i;
     for (i = 0; i < nbSeqs; ++i) {
-        seqDef seq = seqStore->sequencesStart[i];
+        seqDef const seq = seqStore->sequencesStart[i];
         literalsBytes += seq.litLength;
         if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) {
             literalsBytes += 0x10000;
-        }
-    }
+    }   }
     return literalsBytes;
 }
 
 /* Returns match bytes represented in a seqStore */
-static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) {
+static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore)
+{
     size_t matchBytes = 0;
-    size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart;
+    size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
     size_t i;
     for (i = 0; i < nbSeqs; ++i) {
         seqDef seq = seqStore->sequencesStart[i];
         matchBytes += seq.mlBase + MINMATCH;
         if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) {
             matchBytes += 0x10000;
-        }
-    }
+    }   }
     return matchBytes;
 }
 
@@ -3496,6 +3827,7 @@ ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, c
     U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0;  /* [ 0 - 3 ] */
     assert(OFFBASE_IS_REPCODE(offBase));
     if (adjustedRepCode == ZSTD_REP_NUM) {
+        assert(ll0);
         /* litlength == 0 and offCode == 2 implies selection of first repcode - 1
          * This is only valid if it results in a valid offset value, aka > 0.
          * Note : it may happen that `rep[0]==1` in exceptional circumstances.
@@ -3521,14 +3853,16 @@ ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, c
  *        1-3 : repcode 1-3
  *        4+ : real_offset+3
  */
-static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
-                                          seqStore_t* const seqStore, U32 const nbSeq) {
+static void
+ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
+                        const seqStore_t* const seqStore, U32 const nbSeq)
+{
     U32 idx = 0;
     for (; idx < nbSeq; ++idx) {
         seqDef* const seq = seqStore->sequencesStart + idx;
         U32 const ll0 = (seq->litLength == 0);
         U32 const offBase = seq->offBase;
-        assert(seq->offBase > 0);
+        assert(offBase > 0);
         if (OFFBASE_IS_REPCODE(offBase)) {
             U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0);
             U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0);
@@ -3537,7 +3871,7 @@ static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_
              * repcode history.
              */
             if (dRawOffset != cRawOffset) {
-                seq->offBase = cRawOffset + ZSTD_REP_NUM;
+                seq->offBase = OFFSET_TO_OFFBASE(cRawOffset);
             }
         }
         /* Compression repcode history is always updated with values directly from the unmodified seqStore.
@@ -3554,10 +3888,11 @@ static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_
  * Returns the total size of that block (including header) or a ZSTD error code.
  */
 static size_t
-ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, seqStore_t* const seqStore,
+ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc,
+                            const seqStore_t* const seqStore,
                                   repcodes_t* const dRep, repcodes_t* const cRep,
                                   void* dst, size_t dstCapacity,
-                                  const void* src, size_t srcSize,
+                            const void* src, size_t srcSize,
                                   U32 lastBlock, U32 isPartition)
 {
     const U32 rleMaxLength = 25;
@@ -3631,10 +3966,11 @@ typedef struct {
 
 /* Helper function to perform the recursive search for block splits.
  * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half.
- * If advantageous to split, then we recurse down the two sub-blocks. If not, or if an error occurred in estimation, then
- * we do not recurse.
+ * If advantageous to split, then we recurse down the two sub-blocks.
+ * If not, or if an error occurred in estimation, then we do not recurse.
  *
- * Note: The recursion depth is capped by a heuristic minimum number of sequences, defined by MIN_SEQUENCES_BLOCK_SPLITTING.
+ * Note: The recursion depth is capped by a heuristic minimum number of sequences,
+ * defined by MIN_SEQUENCES_BLOCK_SPLITTING.
  * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING).
  * In practice, recursion depth usually doesn't go beyond 4.
  *
@@ -3646,19 +3982,20 @@ static void
 ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx,
                              ZSTD_CCtx* zc, const seqStore_t* origSeqStore)
 {
-    seqStore_t* fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
-    seqStore_t* firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
-    seqStore_t* secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
+    seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
+    seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
+    seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
     size_t estimatedOriginalSize;
     size_t estimatedFirstHalfSize;
     size_t estimatedSecondHalfSize;
     size_t midIdx = (startIdx + endIdx)/2;
 
+    DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx);
+    assert(endIdx >= startIdx);
     if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) {
-        DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences");
+        DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx);
         return;
     }
-    DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx);
     ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx);
     ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx);
     ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx);
@@ -3679,15 +4016,18 @@ ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t end
     }
 }
 
-/* Base recursive function. Populates a table with intra-block partition indices that can improve compression ratio.
+/* Base recursive function.
+ * Populates a table with intra-block partition indices that can improve compression ratio.
  *
- * Returns the number of splits made (which equals the size of the partition table - 1).
+ * @return: number of splits made (which equals the size of the partition table - 1).
  */
 static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq)
 {
-    seqStoreSplits splits = {partitions, 0};
+    seqStoreSplits splits;
+    splits.splitLocations = partitions;
+    splits.idx = 0;
     if (nbSeq <= 4) {
-        DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split");
+        DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq);
         /* Refuse to try and split anything with less than 4 sequences */
         return 0;
     }
@@ -3703,18 +4043,20 @@ static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq)
  * Returns combined size of all blocks (which includes headers), or a ZSTD error code.
  */
 static size_t
-ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity,
-                                       const void* src, size_t blockSize, U32 lastBlock, U32 nbSeq)
+ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc,
+                                    void* dst, size_t dstCapacity,
+                              const void* src, size_t blockSize,
+                                    U32 lastBlock, U32 nbSeq)
 {
     size_t cSize = 0;
     const BYTE* ip = (const BYTE*)src;
     BYTE* op = (BYTE*)dst;
     size_t i = 0;
     size_t srcBytesTotal = 0;
-    U32* partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */
-    seqStore_t* nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
-    seqStore_t* currSeqStore = &zc->blockSplitCtx.currSeqStore;
-    size_t numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
+    U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */
+    seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
+    seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore;
+    size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
 
     /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history
      * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two
@@ -3736,19 +4078,21 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
     ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
     ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t));
 
-    DEBUGLOG(4, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
+    DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
                 (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
                 (unsigned)zc->blockState.matchState.nextToUpdate);
 
     if (numSplits == 0) {
-        size_t cSizeSingleBlock = ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
-                                                                   &dRep, &cRep,
-                                                                    op, dstCapacity,
-                                                                    ip, blockSize,
-                                                                    lastBlock, 0 /* isPartition */);
+        size_t cSizeSingleBlock =
+            ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
+                                            &dRep, &cRep,
+                                            op, dstCapacity,
+                                            ip, blockSize,
+                                            lastBlock, 0 /* isPartition */);
         FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!");
         DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits");
-        assert(cSizeSingleBlock <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize);
+        assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX);
+        assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize);
         return cSizeSingleBlock;
     }
 
@@ -3773,7 +4117,8 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
                                                        op, dstCapacity,
                                                        ip, srcBytes,
                                                        lastBlockEntireSrc, 1 /* isPartition */);
-        DEBUGLOG(5, "Estimated size: %zu actual size: %zu", ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk);
+        DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size",
+                    ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk);
         FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!");
 
         ip += srcBytes;
@@ -3781,10 +4126,10 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
         dstCapacity -= cSizeChunk;
         cSize += cSizeChunk;
         *currSeqStore = *nextSeqStore;
-        assert(cSizeChunk <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize);
+        assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize);
     }
-    /* cRep and dRep may have diverged during the compression. If so, we use the dRep repcodes
-     * for the next block.
+    /* cRep and dRep may have diverged during the compression.
+     * If so, we use the dRep repcodes for the next block.
      */
     ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t));
     return cSize;
@@ -3795,8 +4140,6 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
                               void* dst, size_t dstCapacity,
                               const void* src, size_t srcSize, U32 lastBlock)
 {
-    const BYTE* ip = (const BYTE*)src;
-    BYTE* op = (BYTE*)dst;
     U32 nbSeq;
     size_t cSize;
     DEBUGLOG(4, "ZSTD_compressBlock_splitBlock");
@@ -3807,7 +4150,7 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
         if (bss == ZSTDbss_noCompress) {
             if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
                 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
-            cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock);
+            cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock);
             FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
             DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block");
             return cSize;
@@ -3919,10 +4262,11 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
          *   * cSize >= blockBound(srcSize): We have expanded the block too much so
          *     emit an uncompressed block.
          */
-        {
-            size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
+        {   size_t const cSize =
+                ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
             if (cSize != ERROR(dstSize_tooSmall)) {
-                size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
+                size_t const maxCSize =
+                    srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
                 FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed");
                 if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) {
                     ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
@@ -3930,7 +4274,7 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
                 }
             }
         }
-    }
+    } /* if (bss == ZSTDbss_compress)*/
 
     DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()");
     /* Superblock compression failed, attempt to emit a single no compress block.
@@ -3988,7 +4332,7 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
 *   All blocks will be terminated, all input will be consumed.
 *   Function will issue an error if there is not enough `dstCapacity` to hold the compressed content.
 *   Frame is supposed already started (header already produced)
-*   @return : compressed size, or an error code
+*  @return : compressed size, or an error code
 */
 static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
                                      void* dst, size_t dstCapacity,
@@ -4053,7 +4397,7 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
                     MEM_writeLE24(op, cBlockHeader);
                     cSize += ZSTD_blockHeaderSize;
                 }
-            }
+            }  /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/
 
 
             ip += blockSize;
@@ -4245,7 +4589,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
 {
     ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams;
     assert(!ZSTD_checkCParams(cParams));
-    return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
+    return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog);
 }
 
 size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
@@ -5682,8 +6026,18 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
     params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, ¶ms.cParams);
     params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, ¶ms.cParams);
     params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams);
+    params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
+    params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize);
+    params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel);
 
 #ifdef ZSTD_MULTITHREAD
+    /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */
+    RETURN_ERROR_IF(
+        params.useExternalMatchFinder == 1 && params.nbWorkers >= 1,
+        parameter_combination_unsupported,
+        "External matchfinder isn't supported with nbWorkers >= 1"
+    );
+
     if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
         params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
     }
@@ -5844,13 +6198,20 @@ size_t ZSTD_compressStream2_simpleArgs (
                       const void* src, size_t srcSize, size_t* srcPos,
                             ZSTD_EndDirective endOp)
 {
-    ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
-    ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
+    ZSTD_outBuffer output;
+    ZSTD_inBuffer  input;
+    output.dst = dst;
+    output.size = dstCapacity;
+    output.pos = *dstPos;
+    input.src = src;
+    input.size = srcSize;
+    input.pos = *srcPos;
     /* ZSTD_compressStream2() will check validity of dstPos and srcPos */
-    size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp);
-    *dstPos = output.pos;
-    *srcPos = input.pos;
-    return cErr;
+    {   size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp);
+        *dstPos = output.pos;
+        *srcPos = input.pos;
+        return cErr;
+    }
 }
 
 size_t ZSTD_compress2(ZSTD_CCtx* cctx,
@@ -5873,6 +6234,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
         /* Reset to the original values. */
         cctx->requestedParams.inBufferMode = originalInBufferMode;
         cctx->requestedParams.outBufferMode = originalOutBufferMode;
+
         FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed");
         if (result != 0) {  /* compression not completed, due to lack of output space */
             assert(oPos == dstCapacity);
@@ -5883,29 +6245,25 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
     }
 }
 
-typedef struct {
-    U32 idx;            /* Index in array of ZSTD_Sequence */
-    U32 posInSequence;  /* Position within sequence at idx */
-    size_t posInSrc;    /* Number of bytes given by sequences provided so far */
-} ZSTD_sequencePosition;
-
 /* ZSTD_validateSequence() :
  * @offCode : is presumed to follow format required by ZSTD_storeSeq()
  * @returns a ZSTD error code if sequence is not valid
  */
 static size_t
-ZSTD_validateSequence(U32 offCode, U32 matchLength,
-                      size_t posInSrc, U32 windowLog, size_t dictSize)
+ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch,
+                      size_t posInSrc, U32 windowLog, size_t dictSize, int useExternalMatchFinder)
 {
-    U32 const windowSize = 1 << windowLog;
+    U32 const windowSize = 1u << windowLog;
     /* posInSrc represents the amount of data the decoder would decode up to this point.
      * As long as the amount of data decoded is less than or equal to window size, offsets may be
      * larger than the total length of output decoded in order to reference the dict, even larger than
      * window size. After output surpasses windowSize, we're limited to windowSize offsets again.
      */
     size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize;
-    RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), corruption_detected, "Offset too large!");
-    RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small");
+    size_t const matchLenLowerBound = (minMatch == 3 || useExternalMatchFinder) ? 3 : 4;
+    RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!");
+    /* Validate maxNbSeq is large enough for the given matchLength and minMatch */
+    RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch");
     return 0;
 }
 
@@ -5926,16 +6284,15 @@ static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32
     return offBase;
 }
 
-/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
- * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
- */
-static size_t
+size_t
 ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
                                               ZSTD_sequencePosition* seqPos,
                                         const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
-                                        const void* src, size_t blockSize)
+                                        const void* src, size_t blockSize,
+                                        ZSTD_paramSwitch_e externalRepSearch)
 {
     U32 idx = seqPos->idx;
+    U32 const startIdx = idx;
     BYTE const* ip = (BYTE const*)(src);
     const BYTE* const iend = ip + blockSize;
     repcodes_t updatedRepcodes;
@@ -5953,23 +6310,53 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
     ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
     for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) {
         U32 const litLength = inSeqs[idx].litLength;
-        U32 const ll0 = (litLength == 0);
         U32 const matchLength = inSeqs[idx].matchLength;
-        U32 const offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0);
-        ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
+        U32 offBase;
+
+        if (externalRepSearch == ZSTD_ps_disable) {
+            offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset);
+        } else {
+            U32 const ll0 = (litLength == 0);
+            offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0);
+            ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
+        }
 
         DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
         if (cctx->appliedParams.validateSequences) {
             seqPos->posInSrc += litLength + matchLength;
-            FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, seqPos->posInSrc,
-                                                cctx->appliedParams.cParams.windowLog, dictSize),
+            FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
+                                                cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useExternalMatchFinder),
                                                 "Sequence validation failed");
         }
-        RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation,
+        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
                         "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
         ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
     }
+
+    /* If we skipped repcode search while parsing, we need to update repcodes now */
+    assert(externalRepSearch != ZSTD_ps_auto);
+    assert(idx >= startIdx);
+    if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) {
+        U32* const rep = updatedRepcodes.rep;
+        U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */
+
+        if (lastSeqIdx >= startIdx + 2) {
+            rep[2] = inSeqs[lastSeqIdx - 2].offset;
+            rep[1] = inSeqs[lastSeqIdx - 1].offset;
+            rep[0] = inSeqs[lastSeqIdx].offset;
+        } else if (lastSeqIdx == startIdx + 1) {
+            rep[2] = rep[0];
+            rep[1] = inSeqs[lastSeqIdx - 1].offset;
+            rep[0] = inSeqs[lastSeqIdx].offset;
+        } else {
+            assert(lastSeqIdx == startIdx);
+            rep[2] = rep[1];
+            rep[1] = rep[0];
+            rep[0] = inSeqs[lastSeqIdx].offset;
+        }
+    }
+
     ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t));
 
     if (inSeqs[idx].litLength) {
@@ -5978,27 +6365,15 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
         ip += inSeqs[idx].litLength;
         seqPos->posInSrc += inSeqs[idx].litLength;
     }
-    RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!");
+    RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!");
     seqPos->idx = idx+1;
     return 0;
 }
 
-/* Returns the number of bytes to move the current read position back by.
- * Only non-zero if we ended up splitting a sequence.
- * Otherwise, it may return a ZSTD error if something went wrong.
- *
- * This function will attempt to scan through blockSize bytes
- * represented by the sequences in @inSeqs,
- * storing any (partial) sequences.
- *
- * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
- * avoid splitting a match, or to avoid splitting a match such that it would produce a match
- * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
- */
-static size_t
+size_t
 ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
                                    const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
-                                   const void* src, size_t blockSize)
+                                   const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch)
 {
     U32 idx = seqPos->idx;
     U32 startPosInSequence = seqPos->posInSequence;
@@ -6010,6 +6385,9 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
     U32 bytesAdjustment = 0;
     U32 finalMatchSplit = 0;
 
+    /* TODO(embg) support fast parsing mode in noBlockDelim mode */
+    (void)externalRepSearch;
+
     if (cctx->cdict) {
         dictSize = cctx->cdict->dictContentSize;
     } else if (cctx->prefixDict.dict) {
@@ -6039,7 +6417,6 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
             /* Move to the next sequence */
             endPosInSequence -= currSeq.litLength + currSeq.matchLength;
             startPosInSequence = 0;
-            idx++;
         } else {
             /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence
                does not reach the end of the match. So, we have to split the sequence */
@@ -6085,15 +6462,17 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
 
         if (cctx->appliedParams.validateSequences) {
             seqPos->posInSrc += litLength + matchLength;
-            FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, seqPos->posInSrc,
-                                                   cctx->appliedParams.cParams.windowLog, dictSize),
+            FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
+                                                   cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useExternalMatchFinder),
                                                    "Sequence validation failed");
         }
         DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
-        RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation,
+        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
                         "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
         ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
+        if (!finalMatchSplit)
+            idx++; /* Next Sequence */
     }
     DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
     assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength);
@@ -6116,7 +6495,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
 
 typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
                                        const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
-                                       const void* src, size_t blockSize);
+                                       const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
 static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode)
 {
     ZSTD_sequenceCopier sequenceCopier = NULL;
@@ -6147,13 +6526,13 @@ blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD
         blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength;
         if (end) {
             if (inSeqs[spos].matchLength != 0)
-                RETURN_ERROR(corruption_detected, "delimiter format error : both matchlength and offset must be == 0");
+                RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0");
             break;
         }
         spos++;
     }
     if (!end)
-        RETURN_ERROR(corruption_detected, "Reached end of sequences without finding a block delimiter");
+        RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter");
     return blockSize;
 }
 
@@ -6174,9 +6553,9 @@ static size_t determine_blockSize(ZSTD_sequenceFormat_e mode,
     {   size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos);
         FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters");
         if (explicitBlockSize > blockSize)
-            RETURN_ERROR(corruption_detected, "sequences incorrectly define a too large block");
+            RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block");
         if (explicitBlockSize > remaining)
-            RETURN_ERROR(srcSize_wrong, "sequences define a frame longer than source");
+            RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source");
         return explicitBlockSize;
     }
 }
@@ -6224,7 +6603,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
         ZSTD_resetSeqStore(&cctx->seqStore);
         DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize);
 
-        additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize);
+        additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes);
         FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy");
         blockSize -= additionalByteAdjustment;
 
@@ -6256,7 +6635,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
 
         if (!cctx->isFirstBlock &&
             ZSTD_maybeRLE(&cctx->seqStore) &&
-            ZSTD_isRLE((BYTE const*)src, srcSize)) {
+            ZSTD_isRLE(ip, blockSize)) {
             /* We don't want to emit our first block as a RLE even if it qualifies because
             * doing so will cause the decoder (cli only) to throw a "should consume all input error."
             * This is only an issue for zstd <= v1.4.3
@@ -6493,7 +6872,7 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel,
             cp.targetLength = (unsigned)(-clampedCompressionLevel);
         }
         /* refine parameters based on srcSize & dictSize */
-        return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode);
+        return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto);
     }
 }
 
@@ -6528,3 +6907,21 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH
     if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN;
     return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
 }
+
+void ZSTD_registerExternalMatchFinder(
+    ZSTD_CCtx* zc, void* mState,
+    ZSTD_externalMatchFinder_F* mFinder
+) {
+    if (mFinder != NULL) {
+        ZSTD_externalMatchCtx emctx;
+        emctx.mState = mState;
+        emctx.mFinder = mFinder;
+        emctx.seqBuffer = NULL;
+        emctx.seqBufferCapacity = 0;
+        zc->externalMatchCtx = emctx;
+        zc->requestedParams.useExternalMatchFinder = 1;
+    } else {
+        ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx));
+        zc->requestedParams.useExternalMatchFinder = 0;
+    }
+}
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index baa726f7d..dac7a0aa2 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -118,12 +118,13 @@ typedef struct {
 /** ZSTD_buildBlockEntropyStats() :
  *  Builds entropy for the block.
  *  @return : 0 on success or error code */
-size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
-                             const ZSTD_entropyCTables_t* prevEntropy,
-                                   ZSTD_entropyCTables_t* nextEntropy,
-                             const ZSTD_CCtx_params* cctxParams,
-                                   ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                   void* workspace, size_t wkspSize);
+size_t ZSTD_buildBlockEntropyStats(
+                    const seqStore_t* seqStorePtr,
+                    const ZSTD_entropyCTables_t* prevEntropy,
+                          ZSTD_entropyCTables_t* nextEntropy,
+                    const ZSTD_CCtx_params* cctxParams,
+                          ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                          void* workspace, size_t wkspSize);
 
 /*********************************
 *  Compression internals structs *
@@ -149,6 +150,12 @@ typedef struct {
   size_t capacity;      /* The capacity starting from `seq` pointer */
 } rawSeqStore_t;
 
+typedef struct {
+    U32 idx;            /* Index in array of ZSTD_Sequence */
+    U32 posInSequence;  /* Position within sequence at idx */
+    size_t posInSrc;    /* Number of bytes given by sequences provided so far */
+} ZSTD_sequencePosition;
+
 UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
 
 typedef struct {
@@ -339,6 +346,21 @@ struct ZSTD_CCtx_params_s {
 
     /* Controls prefetching in some dictMatchState matchfinders */
     ZSTD_paramSwitch_e prefetchCDictTables;
+
+    /* Controls whether zstd will fall back to an internal matchfinder
+     * if the external matchfinder returns an error code. */
+    int enableMatchFinderFallback;
+
+    /* Indicates whether an external matchfinder has been referenced.
+     * Users can't set this externally.
+     * It is set internally in ZSTD_registerExternalMatchFinder(). */
+    int useExternalMatchFinder;
+
+    /* Adjust the max block size*/
+    size_t maxBlockSize;
+
+    /* Controls repcode search in external sequence parsing */
+    ZSTD_paramSwitch_e searchForExternalRepcodes;
 };  /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
 
 #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
@@ -370,6 +392,14 @@ typedef struct {
     ZSTD_entropyCTablesMetadata_t entropyMetadata;
 } ZSTD_blockSplitCtx;
 
+/* Context for block-level external matchfinder API */
+typedef struct {
+  void* mState;
+  ZSTD_externalMatchFinder_F* mFinder;
+  ZSTD_Sequence* seqBuffer;
+  size_t seqBufferCapacity;
+} ZSTD_externalMatchCtx;
+
 struct ZSTD_CCtx_s {
     ZSTD_compressionStage_e stage;
     int cParamsChanged;                  /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
@@ -439,6 +469,9 @@ struct ZSTD_CCtx_s {
 
     /* Workspace for block splitter */
     ZSTD_blockSplitCtx blockSplitCtx;
+
+    /* Workspace for external matchfinder */
+    ZSTD_externalMatchCtx externalMatchCtx;
 };
 
 typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
@@ -464,7 +497,7 @@ typedef enum {
                                  * In this mode we take both the source size and the dictionary size
                                  * into account when selecting and adjusting the parameters.
                                  */
-    ZSTD_cpm_unknown = 3,       /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
+    ZSTD_cpm_unknown = 3        /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
                                  * We don't know what these parameters are for. We default to the legacy
                                  * behavior of taking both the source size and the dict size into account
                                  * when selecting and adjusting parameters.
@@ -555,7 +588,7 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
 {
     U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6;
     ZSTD_STATIC_ASSERT(ZSTD_btultra == 8);
-    assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat));
+    assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat));
     return (srcSize >> minlog) + 2;
 }
 
@@ -1142,10 +1175,15 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
                     (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
         assert(blockEndIdx >= loadedDictEnd);
 
-        if (blockEndIdx > loadedDictEnd + maxDist) {
+        if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) {
             /* On reaching window size, dictionaries are invalidated.
              * For simplification, if window size is reached anywhere within next block,
              * the dictionary is invalidated for the full block.
+             *
+             * We also have to invalidate the dictionary if ZSTD_window_update() has detected
+             * non-contiguous segments, which means that loadedDictEnd != window->dictLimit.
+             * loadedDictEnd may be 0, if forceWindow is true, but in that case we never use
+             * dictMatchState, so setting it to NULL is not a problem.
              */
             DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)");
             *loadedDictEndPtr = 0;
@@ -1410,4 +1448,31 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
  */
 void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
 
+/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
+ * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
+ * Note that the block delimiter must include the last literals of the block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
+                                              ZSTD_sequencePosition* seqPos,
+                                        const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+                                        const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
+
+/* Returns the number of bytes to move the current read position back by.
+ * Only non-zero if we ended up splitting a sequence.
+ * Otherwise, it may return a ZSTD error if something went wrong.
+ *
+ * This function will attempt to scan through blockSize bytes
+ * represented by the sequences in @inSeqs,
+ * storing any (partial) sequences.
+ *
+ * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
+ * avoid splitting a match, or to avoid splitting a match such that it would produce a match
+ * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
+                                   const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+                                   const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
+
 #endif /* ZSTD_COMPRESS_H */
diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c
index ea80a45c8..bfd4f11ab 100644
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -24,9 +24,9 @@ static size_t showHexa(const void* src, size_t srcSize)
     const BYTE* const ip = (const BYTE*)src;
     size_t u;
     for (u=0; u= 1);
+    assert(src != NULL);
+    {   const BYTE b = ((const BYTE*)src)[0];
+        size_t p;
+        for (p=1; p31) + (srcSize>4095);
 
-    (void)dstCapacity;  /* dstCapacity already guaranteed to be >=4, hence large enough */
+    assert(dstCapacity >= 4); (void)dstCapacity;
+    assert(allBytesIdentical(src, srcSize));
 
     switch(flSize)
     {
@@ -88,20 +102,41 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
     }
 
     ostart[flSize] = *(const BYTE*)src;
-    DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1);
+    DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1);
     return flSize+1;
 }
 
-size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
-                              ZSTD_hufCTables_t* nextHuf,
-                              ZSTD_strategy strategy, int disableLiteralCompression,
-                              void* dst, size_t dstCapacity,
-                        const void* src, size_t srcSize,
-                              void* entropyWorkspace, size_t entropyWorkspaceSize,
-                        const int bmi2,
-                        unsigned suspectUncompressible, HUF_depth_mode depthMode)
+/* ZSTD_minLiteralsToCompress() :
+ * returns minimal amount of literals
+ * for literal compression to even be attempted.
+ * Minimum is made tighter as compression strategy increases.
+ */
+static size_t
+ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat)
+{
+    assert((int)strategy >= 0);
+    assert((int)strategy <= 9);
+    /* btultra2 : min 8 bytes;
+     * then 2x larger for each successive compression strategy
+     * max threshold 64 bytes */
+    {   int const shift = MIN(9-(int)strategy, 3);
+        size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift;
+        DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc);
+        return mintc;
+    }
+}
+
+size_t ZSTD_compressLiterals (
+                  void* dst, size_t dstCapacity,
+            const void* src, size_t srcSize,
+                  void* entropyWorkspace, size_t entropyWorkspaceSize,
+            const ZSTD_hufCTables_t* prevHuf,
+                  ZSTD_hufCTables_t* nextHuf,
+                  ZSTD_strategy strategy,
+                  int disableLiteralCompression,
+                  int suspectUncompressible,
+                  int bmi2)
 {
-    size_t const minGain = ZSTD_minGain(srcSize, strategy);
     size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
     BYTE*  const ostart = (BYTE*)dst;
     U32 singleStream = srcSize < 256;
@@ -119,16 +154,19 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
     if (disableLiteralCompression)
         return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
 
-    /* small ? don't even attempt compression (speed opt) */
-#   define COMPRESS_LITERALS_SIZE_MIN 63
-    {   size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
-        if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-    }
+    /* if too small, don't even attempt compression (speed opt) */
+    if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode))
+        return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
 
     RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
     {   HUF_repeat repeat = prevHuf->repeatMode;
-        int const preferRepeat = (strategy < ZSTD_lazy) ? srcSize <= 1024 : 0;
-        typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned, HUF_depth_mode);
+        int const flags = 0
+            | (bmi2 ? HUF_flags_bmi2 : 0)
+            | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0)
+            | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0)
+            | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0);
+
+        typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int);
         huf_compress_f huf_compress;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
         huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
@@ -137,23 +175,30 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
                                 HUF_SYMBOLVALUE_MAX, LitHufLog,
                                 entropyWorkspace, entropyWorkspaceSize,
                                 (HUF_CElt*)nextHuf->CTable,
-                                &repeat, preferRepeat,
-                                bmi2, suspectUncompressible, depthMode);
+                                &repeat, flags);
+        DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize);
         if (repeat != HUF_repeat_none) {
             /* reused the existing table */
-            DEBUGLOG(5, "Reusing previous huffman table");
+            DEBUGLOG(5, "reusing statistics from previous huffman block");
             hType = set_repeat;
         }
     }
 
-    if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
-        ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-        return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-    }
+    {   size_t const minGain = ZSTD_minGain(srcSize, strategy);
+        if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
+            ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+            return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+    }   }
     if (cLitSize==1) {
-        ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-        return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
-    }
+        /* A return value of 1 signals that the alphabet consists of a single symbol.
+         * However, in some rare circumstances, it could be the compressed size (a single byte).
+         * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`.
+         * (it's also necessary to not generate statistics).
+         * Therefore, in such a case, actively check that all bytes are identical. */
+        if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) {
+            ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+            return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
+    }   }
 
     if (hType == set_compressed) {
         /* using a newly constructed table */
@@ -164,16 +209,19 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
     switch(lhSize)
     {
     case 3: /* 2 - 2 - 10 - 10 */
+        if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
         {   U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
             MEM_writeLE24(ostart, lhc);
             break;
         }
     case 4: /* 2 - 2 - 14 - 14 */
+        assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
         {   U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18);
             MEM_writeLE32(ostart, lhc);
             break;
         }
     case 5: /* 2 - 2 - 18 - 18 */
+        assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
         {   U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22);
             MEM_writeLE32(ostart, lhc);
             ostart[4] = (BYTE)(cLitSize >> 10);
diff --git a/lib/compress/zstd_compress_literals.h b/lib/compress/zstd_compress_literals.h
index bb260db9b..b060c8ad2 100644
--- a/lib/compress/zstd_compress_literals.h
+++ b/lib/compress/zstd_compress_literals.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,16 +16,24 @@
 
 size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
+/* ZSTD_compressRleLiteralsBlock() :
+ * Conditions :
+ * - All bytes in @src are identical
+ * - dstCapacity >= 4 */
 size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
-/* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
-size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
-                              ZSTD_hufCTables_t* nextHuf,
-                              ZSTD_strategy strategy, int disableLiteralCompression,
-                              void* dst, size_t dstCapacity,
+/* ZSTD_compressLiterals():
+ * @entropyWorkspace: must be aligned on 4-bytes boundaries
+ * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE
+ * @suspectUncompressible: sampling checks, to potentially skip huffman coding
+ */
+size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity,
                         const void* src, size_t srcSize,
                               void* entropyWorkspace, size_t entropyWorkspaceSize,
-                        const int bmi2,
-                        unsigned suspectUncompressible, HUF_depth_mode depthMode);
+                        const ZSTD_hufCTables_t* prevHuf,
+                              ZSTD_hufCTables_t* nextHuf,
+                              ZSTD_strategy strategy, int disableLiteralCompression,
+                              int suspectUncompressible,
+                              int bmi2);
 
 #endif /* ZSTD_COMPRESS_LITERALS_H */
diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c
index 2c1eee567..8872d4d35 100644
--- a/lib/compress/zstd_compress_sequences.c
+++ b/lib/compress/zstd_compress_sequences.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_sequences.h b/lib/compress/zstd_compress_sequences.h
index 7991364c2..4a3a05da9 100644
--- a/lib/compress/zstd_compress_sequences.h
+++ b/lib/compress/zstd_compress_sequences.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c
index eed58e7cf..638c4acbe 100644
--- a/lib/compress/zstd_compress_superblock.c
+++ b/lib/compress/zstd_compress_superblock.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -54,8 +54,6 @@ ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
     symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
     size_t cLitSize = 0;
 
-    (void)bmi2; /* TODO bmi2... */
-
     DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy);
 
     *entropyWritten = 0;
@@ -77,9 +75,9 @@ ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
         DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
     }
 
-    /* TODO bmi2 */
-    {   const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable)
-                                          : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable);
+    {   int const flags = bmi2 ? HUF_flags_bmi2 : 0;
+        const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable, flags)
+                                          : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable, flags);
         op += cSize;
         cLitSize += cSize;
         if (cSize == 0 || ERR_isError(cSize)) {
diff --git a/lib/compress/zstd_compress_superblock.h b/lib/compress/zstd_compress_superblock.h
index 176f9b106..8e494f0d5 100644
--- a/lib/compress/zstd_compress_superblock.h
+++ b/lib/compress/zstd_compress_superblock.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h
index 47afe3dc7..97676693b 100644
--- a/lib/compress/zstd_cwksp.h
+++ b/lib/compress/zstd_cwksp.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -348,7 +348,9 @@ ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase
     if (alloc) {
         alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
         if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
-            __asan_unpoison_memory_region(alloc, bytes);
+            /* We need to keep the redzone poisoned while unpoisoning the bytes that
+             * are actually allocated. */
+            __asan_unpoison_memory_region(alloc, bytes - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE);
         }
     }
 #endif
diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index c2dbd54c1..0ad88ffc7 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_double_fast.h b/lib/compress/zstd_double_fast.h
index 6d8ee8c65..6f0047c4b 100644
--- a/lib/compress/zstd_double_fast.h
+++ b/lib/compress/zstd_double_fast.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index 291173449..5f2c6a2ed 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_fast.h b/lib/compress/zstd_fast.h
index 3bfeb2c5f..9e4236b47 100644
--- a/lib/compress/zstd_fast.h
+++ b/lib/compress/zstd_fast.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index d5a7b5cbf..a24734272 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -759,7 +759,6 @@ size_t ZSTD_HcFindBestMatch(
 ***********************************/
 /* Constants for row-based hash */
 #define ZSTD_ROW_HASH_TAG_OFFSET 16     /* byte offset of hashes in the match state's tagTable from the beginning of a row */
-#define ZSTD_ROW_HASH_TAG_BITS 8        /* nb bits to use for the tag */
 #define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1)
 #define ZSTD_ROW_HASH_MAX_ENTRIES 64    /* absolute maximum number of entries per row, for all configurations */
 
diff --git a/lib/compress/zstd_lazy.h b/lib/compress/zstd_lazy.h
index 150f7b390..3bde67331 100644
--- a/lib/compress/zstd_lazy.h
+++ b/lib/compress/zstd_lazy.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -25,6 +25,8 @@ extern "C" {
  */
 #define ZSTD_LAZY_DDSS_BUCKET_LOG 2
 
+#define ZSTD_ROW_HASH_TAG_BITS 8        /* nb bits to use for the tag */
+
 U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
 void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
 
@@ -116,7 +118,7 @@ size_t ZSTD_compressBlock_lazy2_extDict_row(
 size_t ZSTD_compressBlock_btlazy2_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize);
-        
+
 
 #if defined (__cplusplus)
 }
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index c14c62454..3d74ff19e 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h
index 4e68dbf52..f147021d2 100644
--- a/lib/compress/zstd_ldm.h
+++ b/lib/compress/zstd_ldm.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_ldm_geartab.h b/lib/compress/zstd_ldm_geartab.h
index 647f865be..ef34bc5c9 100644
--- a/lib/compress/zstd_ldm_geartab.h
+++ b/lib/compress/zstd_ldm_geartab.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 800f87e9e..fdd7f9d8b 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,7 +16,7 @@
 #define ZSTD_LITFREQ_ADD    2   /* scaling factor for litFreq, so that frequencies adapt faster to new stats */
 #define ZSTD_MAX_PRICE     (1<<30)
 
-#define ZSTD_PREDEF_THRESHOLD 1024   /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
+#define ZSTD_PREDEF_THRESHOLD 8   /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
 
 
 /*-*************************************
@@ -26,27 +26,35 @@
 #if 0    /* approximation at bit level (for tests) */
 #  define BITCOST_ACCURACY 0
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat, opt) ((void)opt, ZSTD_bitWeight(stat))
+#  define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat))
 #elif 0  /* fractional bit accuracy (for tests) */
 #  define BITCOST_ACCURACY 8
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat))
+#  define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat))
 #else    /* opt==approx, ultra==accurate */
 #  define BITCOST_ACCURACY 8
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat,opt) (opt ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
+#  define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
 #endif
 
+/* ZSTD_bitWeight() :
+ * provide estimated "cost" of a stat in full bits only */
 MEM_STATIC U32 ZSTD_bitWeight(U32 stat)
 {
     return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER);
 }
 
+/* ZSTD_fracWeight() :
+ * provide fractional-bit "cost" of a stat,
+ * using linear interpolation approximation */
 MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
 {
     U32 const stat = rawStat + 1;
     U32 const hb = ZSTD_highbit32(stat);
     U32 const BWeight = hb * BITCOST_MULTIPLIER;
+    /* Fweight was meant for "Fractional weight"
+     * but it's effectively a value between 1 and 2
+     * using fixed point arithmetic */
     U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb;
     U32 const weight = BWeight + FWeight;
     assert(hb + BITCOST_ACCURACY < 31);
@@ -57,7 +65,7 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
 /* debugging function,
  * @return price in bytes as fractional value
  * for debug messages only */
-MEM_STATIC double ZSTD_fCost(U32 price)
+MEM_STATIC double ZSTD_fCost(int price)
 {
     return (double)price / (BITCOST_MULTIPLIER*8);
 }
@@ -88,20 +96,26 @@ static U32 sum_u32(const unsigned table[], size_t nbElts)
     return total;
 }
 
-static U32 ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift)
+typedef enum { base_0possible=0, base_1guaranteed=1 } base_directive_e;
+
+static U32
+ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift, base_directive_e base1)
 {
     U32 s, sum=0;
-    DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", (unsigned)lastEltIndex+1, (unsigned)shift);
+    DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)",
+            (unsigned)lastEltIndex+1, (unsigned)shift );
     assert(shift < 30);
     for (s=0; s> shift);
-        sum += table[s];
+        unsigned const base = base1 ? 1 : (table[s]>0);
+        unsigned const newStat = base + (table[s] >> shift);
+        sum += newStat;
+        table[s] = newStat;
     }
     return sum;
 }
 
 /* ZSTD_scaleStats() :
- * reduce all elements in table is sum too large
+ * reduce all elt frequencies in table if sum too large
  * return the resulting sum of elements */
 static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
 {
@@ -110,7 +124,7 @@ static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
     DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget);
     assert(logTarget < 30);
     if (factor <= 1) return prevsum;
-    return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor));
+    return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed);
 }
 
 /* ZSTD_rescaleFreqs() :
@@ -129,18 +143,22 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
     DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
     optPtr->priceType = zop_dynamic;
 
-    if (optPtr->litLengthSum == 0) {  /* first block : init */
-        if (srcSize <= ZSTD_PREDEF_THRESHOLD) {  /* heuristic */
-            DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef");
+    if (optPtr->litLengthSum == 0) {  /* no literals stats collected -> first block assumed -> init */
+
+        /* heuristic: use pre-defined stats for too small inputs */
+        if (srcSize <= ZSTD_PREDEF_THRESHOLD) {
+            DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD);
             optPtr->priceType = zop_predef;
         }
 
         assert(optPtr->symbolCosts != NULL);
         if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
-            /* huffman table presumed generated by dictionary */
+
+            /* huffman stats covering the full value set : table presumed generated by dictionary */
             optPtr->priceType = zop_dynamic;
 
             if (compressedLiterals) {
+                /* generate literals statistics from huffman table */
                 unsigned lit;
                 assert(optPtr->litFreq != NULL);
                 optPtr->litSum = 0;
@@ -188,13 +206,14 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
                     optPtr->offCodeSum += optPtr->offCodeFreq[of];
             }   }
 
-        } else {  /* not a dictionary */
+        } else {  /* first block, no dictionary */
 
             assert(optPtr->litFreq != NULL);
             if (compressedLiterals) {
+                /* base initial cost of literals on direct frequency within src */
                 unsigned lit = MaxLit;
                 HIST_count_simple(optPtr->litFreq, &lit, src, srcSize);   /* use raw first block to init statistics */
-                optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8);
+                optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible);
             }
 
             {   unsigned const baseLLfreqs[MaxLL+1] = {
@@ -224,10 +243,9 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
                 optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1);
             }
 
-
         }
 
-    } else {   /* new block : re-use previous statistics, scaled down */
+    } else {   /* new block : scale down accumulated statistics */
 
         if (compressedLiterals)
             optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12);
@@ -275,10 +293,11 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
     assert(litLength <= ZSTD_BLOCKSIZE_MAX);
     if (optPtr->priceType == zop_predef)
         return WEIGHT(litLength, optLevel);
-    /* We can't compute the litLength price for sizes >= ZSTD_BLOCKSIZE_MAX
-     * because it isn't representable in the zstd format. So instead just
-     * call it 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. In this case the block
-     * would be all literals.
+
+    /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX
+     * because it isn't representable in the zstd format.
+     * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1.
+     * In such a case, the block would be all literals.
      */
     if (litLength == ZSTD_BLOCKSIZE_MAX)
         return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel);
@@ -292,7 +311,7 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
 }
 
 /* ZSTD_getMatchPrice() :
- * Provides the cost of the match part (offset + matchLength) of a sequence
+ * Provides the cost of the match part (offset + matchLength) of a sequence.
  * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
  * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq()
  * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency)
@@ -308,8 +327,9 @@ ZSTD_getMatchPrice(U32 const offBase,
     U32 const mlBase = matchLength - MINMATCH;
     assert(matchLength >= MINMATCH);
 
-    if (optPtr->priceType == zop_predef)  /* fixed scheme, do not use statistics */
-        return WEIGHT(mlBase, optLevel) + ((16 + offCode) * BITCOST_MULTIPLIER);
+    if (optPtr->priceType == zop_predef)  /* fixed scheme, does not use statistics */
+        return WEIGHT(mlBase, optLevel)
+             + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */
 
     /* dynamic statistics */
     price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel));
@@ -347,7 +367,7 @@ static void ZSTD_updateStats(optState_t* const optPtr,
         optPtr->litLengthSum++;
     }
 
-    /* offset code : expected to follow storeSeq() numeric representation */
+    /* offset code : follows storeSeq() numeric representation */
     {   U32 const offCode = ZSTD_highbit32(offBase);
         assert(offCode <= MaxOff);
         optPtr->offCodeFreq[offCode]++;
@@ -555,16 +575,17 @@ void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
     ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
 }
 
-FORCE_INLINE_TEMPLATE
-U32 ZSTD_insertBtAndGetAllMatches (
-                    ZSTD_match_t* matches,   /* store result (found matches) in this table (presumed large enough) */
-                    ZSTD_matchState_t* ms,
-                    U32* nextToUpdate3,
-                    const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode,
-                    const U32 rep[ZSTD_REP_NUM],
-                    U32 const ll0,   /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
-                    const U32 lengthToBeat,
-                    U32 const mls /* template */)
+FORCE_INLINE_TEMPLATE U32
+ZSTD_insertBtAndGetAllMatches (
+                ZSTD_match_t* matches,  /* store result (found matches) in this table (presumed large enough) */
+                ZSTD_matchState_t* ms,
+                U32* nextToUpdate3,
+                const BYTE* const ip, const BYTE* const iLimit,
+                const ZSTD_dictMode_e dictMode,
+                const U32 rep[ZSTD_REP_NUM],
+                const U32 ll0,  /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
+                const U32 lengthToBeat,
+                const U32 mls /* template */)
 {
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
@@ -1131,7 +1152,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                         U32 const matchPrice = ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel);
                         U32 const sequencePrice = literalsPrice + matchPrice;
                         DEBUGLOG(7, "rPos:%u => set initial price : %.2f",
-                                    pos, ZSTD_fCost(sequencePrice));
+                                    pos, ZSTD_fCost((int)sequencePrice));
                         opt[pos].mlen = pos;
                         opt[pos].off = offBase;
                         opt[pos].litlen = litlen;
@@ -1352,7 +1373,7 @@ size_t ZSTD_compressBlock_btopt(
 /* ZSTD_initStats_ultra():
  * make a first compression pass, just to seed stats with more accurate starting values.
  * only works on first block, with no dictionary and no ldm.
- * this function cannot error, hence its contract must be respected.
+ * this function cannot error out, its narrow contract must be respected.
  */
 static void
 ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
@@ -1371,7 +1392,7 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
 
     ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict);   /* generate stats into ms->opt*/
 
-    /* invalidate first scan from history */
+    /* invalidate first scan from history, only keep entropy stats */
     ZSTD_resetSeqStore(seqStore);
     ms->window.base -= srcSize;
     ms->window.dictLimit += (U32)srcSize;
@@ -1395,20 +1416,20 @@ size_t ZSTD_compressBlock_btultra2(
     U32 const curr = (U32)((const BYTE*)src - ms->window.base);
     DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
 
-    /* 2-pass strategy:
+    /* 2-passes strategy:
      * this strategy makes a first pass over first block to collect statistics
-     * and seed next round's statistics with it.
-     * After 1st pass, function forgets everything, and starts a new block.
+     * in order to seed next round's statistics with it.
+     * After 1st pass, function forgets history, and starts a new block.
      * Consequently, this can only work if no data has been previously loaded in tables,
      * aka, no dictionary, no prefix, no ldm preprocessing.
      * The compression ratio gain is generally small (~0.5% on first block),
-     * the cost is 2x cpu time on first block. */
+    ** the cost is 2x cpu time on first block. */
     assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
     if ( (ms->opt.litLengthSum==0)   /* first block */
       && (seqStore->sequences == seqStore->sequencesStart)  /* no ldm */
       && (ms->window.dictLimit == ms->window.lowLimit)   /* no dictionary */
-      && (curr == ms->window.dictLimit)   /* start of frame, nothing already loaded nor skipped */
-      && (srcSize > ZSTD_PREDEF_THRESHOLD)
+      && (curr == ms->window.dictLimit)    /* start of frame, nothing already loaded nor skipped */
+      && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */
       ) {
         ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
     }
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index 627255f53..342e5a311 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 0c10eb603..7a2c71720 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 271eb1ac7..ed4dc0e99 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index c6fd92860..c2d1f633a 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -1,7 +1,7 @@
 /* ******************************************************************
  * huff0 huffman decoder,
  * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -19,7 +19,6 @@
 #include "../common/compiler.h"
 #include "../common/bitstream.h"  /* BIT_* */
 #include "../common/fse.h"        /* to compress headers */
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "../common/error_private.h"
 #include "../common/zstd_internal.h"
@@ -44,10 +43,14 @@
 #error "Cannot force the use of the X1 and X2 decoders at the same time!"
 #endif
 
-#if ZSTD_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2
-# define HUF_ASM_X86_64_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
+/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is
+ * supported at runtime, so we can add the BMI2 target attribute.
+ * When it is disabled, we will still get BMI2 if it is enabled statically.
+ */
+#if DYNAMIC_BMI2
+# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
 #else
-# define HUF_ASM_X86_64_BMI2_ATTRS
+# define HUF_FAST_BMI2_ATTRS
 #endif
 
 #ifdef __cplusplus
@@ -57,18 +60,12 @@
 #endif
 #define HUF_ASM_DECL HUF_EXTERN_C
 
-#if DYNAMIC_BMI2 || (ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
+#if DYNAMIC_BMI2
 # define HUF_NEED_BMI2_FUNCTION 1
 #else
 # define HUF_NEED_BMI2_FUNCTION 0
 #endif
 
-#if !(ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
-# define HUF_NEED_DEFAULT_FUNCTION 1
-#else
-# define HUF_NEED_DEFAULT_FUNCTION 0
-#endif
-
 /* **************************************************************
 *  Error Management
 ****************************************************************/
@@ -85,6 +82,11 @@
 /* **************************************************************
 *  BMI2 Variant Wrappers
 ****************************************************************/
+typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize,
+                                              const void *cSrc,
+                                              size_t cSrcSize,
+                                              const HUF_DTable *DTable);
+
 #if DYNAMIC_BMI2
 
 #define HUF_DGEN(fn)                                                        \
@@ -106,9 +108,9 @@
     }                                                                       \
                                                                             \
     static size_t fn(void* dst, size_t dstSize, void const* cSrc,           \
-                     size_t cSrcSize, HUF_DTable const* DTable, int bmi2)   \
+                     size_t cSrcSize, HUF_DTable const* DTable, int flags)  \
     {                                                                       \
-        if (bmi2) {                                                         \
+        if (flags & HUF_flags_bmi2) {                                       \
             return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);         \
         }                                                                   \
         return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable);          \
@@ -118,9 +120,9 @@
 
 #define HUF_DGEN(fn)                                                        \
     static size_t fn(void* dst, size_t dstSize, void const* cSrc,           \
-                     size_t cSrcSize, HUF_DTable const* DTable, int bmi2)   \
+                     size_t cSrcSize, HUF_DTable const* DTable, int flags)  \
     {                                                                       \
-        (void)bmi2;                                                         \
+        (void)flags;                                                        \
         return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable);             \
     }
 
@@ -139,15 +141,28 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
     return dtd;
 }
 
-#if ZSTD_ENABLE_ASM_X86_64_BMI2
-
-static size_t HUF_initDStream(BYTE const* ip) {
+static size_t HUF_initFastDStream(BYTE const* ip) {
     BYTE const lastByte = ip[7];
     size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
     size_t const value = MEM_readLEST(ip) | 1;
     assert(bitsConsumed <= 8);
+    assert(sizeof(size_t) == 8);
     return value << bitsConsumed;
 }
+
+
+/**
+ * The input/output arguments to the Huffman fast decoding loop:
+ *
+ * ip [in/out] - The input pointers, must be updated to reflect what is consumed.
+ * op [in/out] - The output pointers, must be updated to reflect what is written.
+ * bits [in/out] - The bitstream containers, must be updated to reflect the current state.
+ * dt [in] - The decoding table.
+ * ilimit [in] - The input limit, stop when any input pointer is below ilimit.
+ * oend [in] - The end of the output stream. op[3] must not cross oend.
+ * iend [in] - The end of each input stream. ip[i] may cross iend[i],
+ *             as long as it is above ilimit, but that indicates corruption.
+ */
 typedef struct {
     BYTE const* ip[4];
     BYTE* op[4];
@@ -156,15 +171,17 @@ typedef struct {
     BYTE const* ilimit;
     BYTE* oend;
     BYTE const* iend[4];
-} HUF_DecompressAsmArgs;
+} HUF_DecompressFastArgs;
+
+typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*);
 
 /**
- * Initializes args for the asm decoding loop.
- * @returns 0 on success
- *          1 if the fallback implementation should be used.
+ * Initializes args for the fast decoding loop.
+ * @returns 1 on success
+ *          0 if the fallback implementation should be used.
  *          Or an error code on failure.
  */
-static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
+static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
 {
     void const* dt = DTable + 1;
     U32 const dtLog = HUF_getDTableDesc(DTable).tableLog;
@@ -173,9 +190,11 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
 
     BYTE* const oend = (BYTE*)dst + dstSize;
 
-    /* The following condition is false on x32 platform,
-     * but HUF_asm is not compatible with this ABI */
-    if (!(MEM_isLittleEndian() && !MEM_32bits())) return 1;
+    /* The fast decoding loop assumes 64-bit little-endian.
+     * This condition is false on x32.
+     */
+    if (!MEM_isLittleEndian() || MEM_32bits())
+        return 0;
 
     /* strict minimum : jump table + 1 byte per stream */
     if (srcSize < 10)
@@ -186,7 +205,7 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
      * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder.
      */
     if (dtLog != HUF_DECODER_FAST_TABLELOG)
-        return 1;
+        return 0;
 
     /* Read the jump table. */
     {
@@ -200,13 +219,13 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
         args->iend[2] = args->iend[1] + length2;
         args->iend[3] = args->iend[2] + length3;
 
-        /* HUF_initDStream() requires this, and this small of an input
+        /* HUF_initFastDStream() requires this, and this small of an input
          * won't benefit from the ASM loop anyways.
          * length1 must be >= 16 so that ip[0] >= ilimit before the loop
          * starts.
          */
         if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8)
-            return 1;
+            return 0;
         if (length4 > srcSize) return ERROR(corruption_detected);   /* overflow */
     }
     /* ip[] contains the position that is currently loaded into bits[]. */
@@ -223,7 +242,7 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
 
     /* No point to call the ASM loop for tiny outputs. */
     if (args->op[3] >= oend)
-        return 1;
+        return 0;
 
     /* bits[] is the bit container.
         * It is read from the MSB down to the LSB.
@@ -232,10 +251,10 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
         * set, so that CountTrailingZeros(bits[]) can be used
         * to count how many bits we've consumed.
         */
-    args->bits[0] = HUF_initDStream(args->ip[0]);
-    args->bits[1] = HUF_initDStream(args->ip[1]);
-    args->bits[2] = HUF_initDStream(args->ip[2]);
-    args->bits[3] = HUF_initDStream(args->ip[3]);
+    args->bits[0] = HUF_initFastDStream(args->ip[0]);
+    args->bits[1] = HUF_initFastDStream(args->ip[1]);
+    args->bits[2] = HUF_initFastDStream(args->ip[2]);
+    args->bits[3] = HUF_initFastDStream(args->ip[3]);
 
     /* If ip[] >= ilimit, it is guaranteed to be safe to
         * reload bits[]. It may be beyond its section, but is
@@ -246,10 +265,10 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
     args->oend = oend;
     args->dt = dt;
 
-    return 0;
+    return 1;
 }
 
-static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs const* args, int stream, BYTE* segmentEnd)
+static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd)
 {
     /* Validate that we haven't overwritten. */
     if (args->op[stream] > segmentEnd)
@@ -264,7 +283,7 @@ static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs
 
     /* Construct the BIT_DStream_t. */
     assert(sizeof(size_t) == 8);
-    bit->bitContainer = MEM_readLE64(args->ip[stream]);
+    bit->bitContainer = MEM_readLEST(args->ip[stream]);
     bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]);
     bit->start = (const char*)args->iend[0];
     bit->limitPtr = bit->start + sizeof(size_t);
@@ -272,7 +291,6 @@ static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs
 
     return 0;
 }
-#endif
 
 
 #ifndef HUF_FORCE_DECOMPRESS_X2
@@ -289,10 +307,11 @@ typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1;   /* single-symbol decodi
 static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) {
     U64 D4;
     if (MEM_isLittleEndian()) {
-        D4 = (symbol << 8) + nbBits;
+        D4 = (U64)((symbol << 8) + nbBits);
     } else {
-        D4 = symbol + (nbBits << 8);
+        D4 = (U64)(symbol + (nbBits << 8));
     }
+    assert(D4 < (1U << 16));
     D4 *= 0x0001000100010001ULL;
     return D4;
 }
@@ -335,13 +354,7 @@ typedef struct {
         BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
 } HUF_ReadDTableX1_Workspace;
 
-
-size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize)
-{
-    return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
-size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2)
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags)
 {
     U32 tableLog = 0;
     U32 nbSymbols = 0;
@@ -356,7 +369,7 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
     DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
     /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */   /* is not necessary, even though some analyzer complain ... */
 
-    iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2);
+    iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags);
     if (HUF_isError(iSize)) return iSize;
 
 
@@ -383,9 +396,8 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
      * rankStart[0] is not filled because there are no entries in the table for
      * weight 0.
      */
-    {
-        int n;
-        int nextRankStart = 0;
+    {   int n;
+        U32 nextRankStart = 0;
         int const unroll = 4;
         int const nLimit = (int)nbSymbols - unroll + 1;
         for (n=0; n<(int)tableLog+1; n++) {
@@ -412,10 +424,9 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
      * We can switch based on the length to a different inner loop which is
      * optimized for that particular case.
      */
-    {
-        U32 w;
-        int symbol=wksp->rankVal[0];
-        int rankStart=0;
+    {   U32 w;
+        int symbol = wksp->rankVal[0];
+        int rankStart = 0;
         for (w=1; wrankVal[w];
             int const length = (1 << w) >> 1;
@@ -525,7 +536,7 @@ HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, cons
     while (p < pEnd)
         HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
 
-    return pEnd-pStart;
+    return (size_t)(pEnd-pStart);
 }
 
 FORCE_INLINE_TEMPLATE size_t
@@ -551,6 +562,10 @@ HUF_decompress1X1_usingDTable_internal_body(
     return dstSize;
 }
 
+/* HUF_decompress4X1_usingDTable_internal_body():
+ * Conditions :
+ * @dstSize >= 6
+ */
 FORCE_INLINE_TEMPLATE size_t
 HUF_decompress4X1_usingDTable_internal_body(
           void* dst,  size_t dstSize,
@@ -594,6 +609,7 @@ HUF_decompress4X1_usingDTable_internal_body(
 
         if (length4 > cSrcSize) return ERROR(corruption_detected);   /* overflow */
         if (opStart4 > oend) return ERROR(corruption_detected);      /* overflow */
+        if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
         CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
         CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
         CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
@@ -656,38 +672,142 @@ size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, vo
 }
 #endif
 
-#if HUF_NEED_DEFAULT_FUNCTION
 static
 size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
                     size_t cSrcSize, HUF_DTable const* DTable) {
     return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
 }
-#endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2
 
-HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN;
+HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
 
-static HUF_ASM_X86_64_BMI2_ATTRS
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+    U64 bits[4];
+    BYTE const* ip[4];
+    BYTE* op[4];
+    U16 const* const dtable = (U16 const*)args->dt;
+    BYTE* const oend = args->oend;
+    BYTE const* const ilimit = args->ilimit;
+
+    /* Copy the arguments to local variables */
+    ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+    ZSTD_memcpy(&ip, &args->ip, sizeof(ip));
+    ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+    assert(MEM_isLittleEndian());
+    assert(!MEM_32bits());
+
+    for (;;) {
+        BYTE* olimit;
+        int stream;
+        int symbol;
+
+        /* Assert loop preconditions */
+#ifndef NDEBUG
+        for (stream = 0; stream < 4; ++stream) {
+            assert(op[stream] <= (stream == 3 ? oend : op[stream + 1]));
+            assert(ip[stream] >= ilimit);
+        }
+#endif
+        /* Compute olimit */
+        {
+            /* Each iteration produces 5 output symbols per stream */
+            size_t const oiters = (size_t)(oend - op[3]) / 5;
+            /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes
+             * per stream.
+             */
+            size_t const iiters = (size_t)(ip[0] - ilimit) / 7;
+            /* We can safely run iters iterations before running bounds checks */
+            size_t const iters = MIN(oiters, iiters);
+            size_t const symbols = iters * 5;
+
+            /* We can simply check that op[3] < olimit, instead of checking all
+             * of our bounds, since we can't hit the other bounds until we've run
+             * iters iterations, which only happens when op[3] == olimit.
+             */
+            olimit = op[3] + symbols;
+
+            /* Exit fast decoding loop once we get close to the end. */
+            if (op[3] + 20 > olimit)
+                break;
+
+            /* Exit the decoding loop if any input pointer has crossed the
+             * previous one. This indicates corruption, and a precondition
+             * to our loop is that ip[i] >= ip[0].
+             */
+            for (stream = 1; stream < 4; ++stream) {
+                if (ip[stream] < ip[stream - 1])
+                    goto _out;
+            }
+        }
+
+#ifndef NDEBUG
+        for (stream = 1; stream < 4; ++stream) {
+            assert(ip[stream] >= ip[stream - 1]);
+        }
+#endif
+
+        do {
+            /* Decode 5 symbols in each of the 4 streams */
+            for (symbol = 0; symbol < 5; ++symbol) {
+                for (stream = 0; stream < 4; ++stream) {
+                    int const index = (int)(bits[stream] >> 53);
+                    int const entry = (int)dtable[index];
+                    bits[stream] <<= (entry & 63);
+                    op[stream][symbol] = (BYTE)((entry >> 8) & 0xFF);
+                }
+            }
+            /* Reload the bitstreams */
+            for (stream = 0; stream < 4; ++stream) {
+                int const ctz = ZSTD_countTrailingZeros64(bits[stream]);
+                int const nbBits = ctz & 7;
+                int const nbBytes = ctz >> 3;
+                op[stream] += 5;
+                ip[stream] -= nbBytes;
+                bits[stream] = MEM_read64(ip[stream]) | 1;
+                bits[stream] <<= nbBits;
+            }
+        } while (op[3] < olimit);
+    }
+
+_out:
+
+    /* Save the final values of each of the state variables back to args. */
+    ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+    ZSTD_memcpy(&args->ip, &ip, sizeof(ip));
+    ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+/**
+ * @returns @p dstSize on success (>= 6)
+ *          0 if the fallback implementation should be used
+ *          An error if an error occurred
+ */
+static HUF_FAST_BMI2_ATTRS
 size_t
-HUF_decompress4X1_usingDTable_internal_bmi2_asm(
+HUF_decompress4X1_usingDTable_internal_fast(
           void* dst,  size_t dstSize,
     const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
+    const HUF_DTable* DTable,
+    HUF_DecompressFastLoopFn loopFn)
 {
     void const* dt = DTable + 1;
     const BYTE* const iend = (const BYTE*)cSrc + 6;
     BYTE* const oend = (BYTE*)dst + dstSize;
-    HUF_DecompressAsmArgs args;
-    {
-        size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
-        FORWARD_IF_ERROR(ret, "Failed to init asm args");
-        if (ret != 0)
-            return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+    HUF_DecompressFastArgs args;
+    {   size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+        FORWARD_IF_ERROR(ret, "Failed to init fast loop args");
+        if (ret == 0)
+            return 0;
     }
 
     assert(args.ip[0] >= args.ilimit);
-    HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(&args);
+    loopFn(&args);
 
     /* Our loop guarantees that ip[] >= ilimit and that we haven't
     * overwritten any op[].
@@ -700,8 +820,7 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm(
     (void)iend;
 
     /* finish bit streams one by one. */
-    {
-        size_t const segmentSize = (dstSize+3) / 4;
+    {   size_t const segmentSize = (dstSize+3) / 4;
         BYTE* segmentEnd = (BYTE*)dst;
         int i;
         for (i = 0; i < 4; ++i) {
@@ -718,97 +837,59 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm(
     }
 
     /* decoded size */
+    assert(dstSize != 0);
     return dstSize;
 }
-#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
-
-typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize,
-                                               const void *cSrc,
-                                               size_t cSrcSize,
-                                               const HUF_DTable *DTable);
 
 HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
 
 static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
-                    size_t cSrcSize, HUF_DTable const* DTable, int bmi2)
+                    size_t cSrcSize, HUF_DTable const* DTable, int flags)
 {
+    HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default;
+    HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop;
+
 #if DYNAMIC_BMI2
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
+        fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2;
 # if ZSTD_ENABLE_ASM_X86_64_BMI2
-        return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-# else
-        return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+        if (!(flags & HUF_flags_disableAsm)) {
+            loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+        }
 # endif
+    } else {
+        return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
     }
-#else
-    (void)bmi2;
 #endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
-    return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-#else
-    return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
+    if (!(flags & HUF_flags_disableAsm)) {
+        loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+    }
 #endif
+
+    if (!(flags & HUF_flags_disableFast)) {
+        size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+        if (ret != 0)
+            return ret;
+    }
+    return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
 }
 
-
-size_t HUF_decompress1X1_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 0) return ERROR(GENERIC);
-    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
-size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
+static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
                                    const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
+                                   void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
-    size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize);
+    size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0);
+    return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
 }
 
-
-size_t HUF_decompress4X1_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 0) return ERROR(GENERIC);
-    return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
-static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
-                                   const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize, int bmi2)
-{
-    const BYTE* ip = (const BYTE*) cSrc;
-
-    size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
-    if (HUF_isError(hSize)) return hSize;
-    if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
-    ip += hSize; cSrcSize -= hSize;
-
-    return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
-}
-
-size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
-                                   const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
-{
-    return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0);
-}
-
-
 #endif /* HUF_FORCE_DECOMPRESS_X2 */
 
 
@@ -991,7 +1072,7 @@ static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32
 
 static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
                            const sortedSymbol_t* sortedList,
-                           const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
+                           const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight,
                            const U32 nbBitsBaseline)
 {
     U32* const rankVal = rankValOrigin[0];
@@ -1046,14 +1127,7 @@ typedef struct {
 
 size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
                        const void* src, size_t srcSize,
-                             void* workSpace, size_t wkspSize)
-{
-    return HUF_readDTableX2_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
-size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable,
-                       const void* src, size_t srcSize,
-                             void* workSpace, size_t wkspSize, int bmi2)
+                             void* workSpace, size_t wkspSize, int flags)
 {
     U32 tableLog, maxW, nbSymbols;
     DTableDesc dtd = HUF_getDTableDesc(DTable);
@@ -1075,7 +1149,7 @@ size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable,
     if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
     /* ZSTD_memset(weightList, 0, sizeof(weightList)); */  /* is not necessary, even though some analyzer complain ... */
 
-    iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), bmi2);
+    iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags);
     if (HUF_isError(iSize)) return iSize;
 
     /* check result */
@@ -1246,6 +1320,11 @@ HUF_decompress1X2_usingDTable_internal_body(
     /* decoded size */
     return dstSize;
 }
+
+/* HUF_decompress4X2_usingDTable_internal_body():
+ * Conditions:
+ * @dstSize >= 6
+ */
 FORCE_INLINE_TEMPLATE size_t
 HUF_decompress4X2_usingDTable_internal_body(
           void* dst,  size_t dstSize,
@@ -1286,8 +1365,9 @@ HUF_decompress4X2_usingDTable_internal_body(
         DTableDesc const dtd = HUF_getDTableDesc(DTable);
         U32 const dtLog = dtd.tableLog;
 
-        if (length4 > cSrcSize) return ERROR(corruption_detected);   /* overflow */
-        if (opStart4 > oend) return ERROR(corruption_detected);      /* overflow */
+        if (length4 > cSrcSize) return ERROR(corruption_detected);  /* overflow */
+        if (opStart4 > oend) return ERROR(corruption_detected);     /* overflow */
+        if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
         CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
         CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
         CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
@@ -1372,36 +1452,177 @@ size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, vo
 }
 #endif
 
-#if HUF_NEED_DEFAULT_FUNCTION
 static
 size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
                     size_t cSrcSize, HUF_DTable const* DTable) {
     return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
 }
-#endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2
 
-HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN;
+HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
 
-static HUF_ASM_X86_64_BMI2_ATTRS size_t
-HUF_decompress4X2_usingDTable_internal_bmi2_asm(
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+    U64 bits[4];
+    BYTE const* ip[4];
+    BYTE* op[4];
+    BYTE* oend[4];
+    HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt;
+    BYTE const* const ilimit = args->ilimit;
+
+    /* Copy the arguments to local registers. */
+    ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+    ZSTD_memcpy(&ip, &args->ip, sizeof(ip));
+    ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+    oend[0] = op[1];
+    oend[1] = op[2];
+    oend[2] = op[3];
+    oend[3] = args->oend;
+
+    assert(MEM_isLittleEndian());
+    assert(!MEM_32bits());
+
+    for (;;) {
+        BYTE* olimit;
+        int stream;
+        int symbol;
+
+        /* Assert loop preconditions */
+#ifndef NDEBUG
+        for (stream = 0; stream < 4; ++stream) {
+            assert(op[stream] <= oend[stream]);
+            assert(ip[stream] >= ilimit);
+        }
+#endif
+        /* Compute olimit */
+        {
+            /* Each loop does 5 table lookups for each of the 4 streams.
+             * Each table lookup consumes up to 11 bits of input, and produces
+             * up to 2 bytes of output.
+             */
+            /* We can consume up to 7 bytes of input per iteration per stream.
+             * We also know that each input pointer is >= ip[0]. So we can run
+             * iters loops before running out of input.
+             */
+            size_t iters = (size_t)(ip[0] - ilimit) / 7;
+            /* Each iteration can produce up to 10 bytes of output per stream.
+             * Each output stream my advance at different rates. So take the
+             * minimum number of safe iterations among all the output streams.
+             */
+            for (stream = 0; stream < 4; ++stream) {
+                size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10;
+                iters = MIN(iters, oiters);
+            }
+
+            /* Each iteration produces at least 5 output symbols. So until
+             * op[3] crosses olimit, we know we haven't executed iters
+             * iterations yet. This saves us maintaining an iters counter,
+             * at the expense of computing the remaining # of iterations
+             * more frequently.
+             */
+            olimit = op[3] + (iters * 5);
+
+            /* Exit the fast decoding loop if we are too close to the end. */
+            if (op[3] + 10 > olimit)
+                break;
+
+            /* Exit the decoding loop if any input pointer has crossed the
+             * previous one. This indicates corruption, and a precondition
+             * to our loop is that ip[i] >= ip[0].
+             */
+            for (stream = 1; stream < 4; ++stream) {
+                if (ip[stream] < ip[stream - 1])
+                    goto _out;
+            }
+        }
+
+#ifndef NDEBUG
+        for (stream = 1; stream < 4; ++stream) {
+            assert(ip[stream] >= ip[stream - 1]);
+        }
+#endif
+
+        do {
+            /* Do 5 table lookups for each of the first 3 streams */
+            for (symbol = 0; symbol < 5; ++symbol) {
+                for (stream = 0; stream < 3; ++stream) {
+                    int const index = (int)(bits[stream] >> 53);
+                    HUF_DEltX2 const entry = dtable[index];
+                    MEM_write16(op[stream], entry.sequence);
+                    bits[stream] <<= (entry.nbBits);
+                    op[stream] += (entry.length);
+                }
+            }
+            /* Do 1 table lookup from the final stream */
+            {
+                int const index = (int)(bits[3] >> 53);
+                HUF_DEltX2 const entry = dtable[index];
+                MEM_write16(op[3], entry.sequence);
+                bits[3] <<= (entry.nbBits);
+                op[3] += (entry.length);
+            }
+            /* Do 4 table lookups from the final stream & reload bitstreams */
+            for (stream = 0; stream < 4; ++stream) {
+                /* Do a table lookup from the final stream.
+                 * This is interleaved with the reloading to reduce register
+                 * pressure. This shouldn't be necessary, but compilers can
+                 * struggle with codegen with high register pressure.
+                 */
+                {
+                    int const index = (int)(bits[3] >> 53);
+                    HUF_DEltX2 const entry = dtable[index];
+                    MEM_write16(op[3], entry.sequence);
+                    bits[3] <<= (entry.nbBits);
+                    op[3] += (entry.length);
+                }
+                /* Reload the bistreams. The final bitstream must be reloaded
+                 * after the 5th symbol was decoded.
+                 */
+                {
+                    int const ctz = ZSTD_countTrailingZeros64(bits[stream]);
+                    int const nbBits = ctz & 7;
+                    int const nbBytes = ctz >> 3;
+                    ip[stream] -= nbBytes;
+                    bits[stream] = MEM_read64(ip[stream]) | 1;
+                    bits[stream] <<= nbBits;
+                }
+            }
+        } while (op[3] < olimit);
+    }
+
+_out:
+
+    /* Save the final values of each of the state variables back to args. */
+    ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+    ZSTD_memcpy(&args->ip, &ip, sizeof(ip));
+    ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+
+static HUF_FAST_BMI2_ATTRS size_t
+HUF_decompress4X2_usingDTable_internal_fast(
           void* dst,  size_t dstSize,
     const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable) {
+    const HUF_DTable* DTable,
+    HUF_DecompressFastLoopFn loopFn) {
     void const* dt = DTable + 1;
     const BYTE* const iend = (const BYTE*)cSrc + 6;
     BYTE* const oend = (BYTE*)dst + dstSize;
-    HUF_DecompressAsmArgs args;
+    HUF_DecompressFastArgs args;
     {
-        size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+        size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
         FORWARD_IF_ERROR(ret, "Failed to init asm args");
-        if (ret != 0)
-            return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+        if (ret == 0)
+            return 0;
     }
 
     assert(args.ip[0] >= args.ilimit);
-    HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(&args);
+    loopFn(&args);
 
     /* note : op4 already verified within main loop */
     assert(args.ip[0] >= iend);
@@ -1432,91 +1653,72 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm(
     /* decoded size */
     return dstSize;
 }
-#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
 
 static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
-                    size_t cSrcSize, HUF_DTable const* DTable, int bmi2)
+                    size_t cSrcSize, HUF_DTable const* DTable, int flags)
 {
+    HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default;
+    HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop;
+
 #if DYNAMIC_BMI2
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
+        fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2;
 # if ZSTD_ENABLE_ASM_X86_64_BMI2
-        return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-# else
-        return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+        if (!(flags & HUF_flags_disableAsm)) {
+            loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+        }
 # endif
+    } else {
+        return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
     }
-#else
-    (void)bmi2;
 #endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
-    return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-#else
-    return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
+    if (!(flags & HUF_flags_disableAsm)) {
+        loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+    }
 #endif
+
+    if (!(flags & HUF_flags_disableFast)) {
+        size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+        if (ret != 0)
+            return ret;
+    }
+    return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
 }
 
 HUF_DGEN(HUF_decompress1X2_usingDTable_internal)
 
-size_t HUF_decompress1X2_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 1) return ERROR(GENERIC);
-    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
 size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
                                    const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
+                                   void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
     size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize,
-                                               workSpace, wkspSize);
+                                               workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0);
+    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags);
 }
 
-
-size_t HUF_decompress4X2_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 1) return ERROR(GENERIC);
-    return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
-static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
+static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
                                    const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize, int bmi2)
+                                   void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
     size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize,
-                                         workSpace, wkspSize);
+                                         workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
+    return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
 }
 
-size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
-                                   const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
-{
-    return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
-
 #endif /* HUF_FORCE_DECOMPRESS_X1 */
 
 
@@ -1524,44 +1726,6 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
 /* Universal decompression selectors */
 /* ***********************************/
 
-size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize,
-                                    const void* cSrc, size_t cSrcSize,
-                                    const HUF_DTable* DTable)
-{
-    DTableDesc const dtd = HUF_getDTableDesc(DTable);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-    (void)dtd;
-    assert(dtd.tableType == 0);
-    return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-    (void)dtd;
-    assert(dtd.tableType == 1);
-    return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#else
-    return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
-                           HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#endif
-}
-
-size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
-                                    const void* cSrc, size_t cSrcSize,
-                                    const HUF_DTable* DTable)
-{
-    DTableDesc const dtd = HUF_getDTableDesc(DTable);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-    (void)dtd;
-    assert(dtd.tableType == 0);
-    return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-    (void)dtd;
-    assert(dtd.tableType == 1);
-    return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#else
-    return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
-                           HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#endif
-}
-
 
 #if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
 typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
@@ -1616,36 +1780,9 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
 #endif
 }
 
-
-size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
-                                     size_t dstSize, const void* cSrc,
-                                     size_t cSrcSize, void* workSpace,
-                                     size_t wkspSize)
-{
-    /* validation checks */
-    if (dstSize == 0) return ERROR(dstSize_tooSmall);
-    if (cSrcSize == 0) return ERROR(corruption_detected);
-
-    {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-        (void)algoNb;
-        assert(algoNb == 0);
-        return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-        (void)algoNb;
-        assert(algoNb == 1);
-        return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#else
-        return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                            cSrcSize, workSpace, wkspSize):
-                        HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#endif
-    }
-}
-
 size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
                                   const void* cSrc, size_t cSrcSize,
-                                  void* workSpace, size_t wkspSize)
+                                  void* workSpace, size_t wkspSize, int flags)
 {
     /* validation checks */
     if (dstSize == 0) return ERROR(dstSize_tooSmall);
@@ -1658,71 +1795,71 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
         (void)algoNb;
         assert(algoNb == 0);
         return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize);
+                                cSrcSize, workSpace, wkspSize, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
         (void)algoNb;
         assert(algoNb == 1);
         return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize);
+                                cSrcSize, workSpace, wkspSize, flags);
 #else
         return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize):
+                                cSrcSize, workSpace, wkspSize, flags):
                         HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize);
+                                cSrcSize, workSpace, wkspSize, flags);
 #endif
     }
 }
 
 
-size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
 {
     DTableDesc const dtd = HUF_getDTableDesc(DTable);
 #if defined(HUF_FORCE_DECOMPRESS_X1)
     (void)dtd;
     assert(dtd.tableType == 0);
-    return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
     (void)dtd;
     assert(dtd.tableType == 1);
-    return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #else
-    return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
-                           HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+                           HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #endif
 }
 
 #ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
-    size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+    size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
+    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
 }
 #endif
 
-size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
 {
     DTableDesc const dtd = HUF_getDTableDesc(DTable);
 #if defined(HUF_FORCE_DECOMPRESS_X1)
     (void)dtd;
     assert(dtd.tableType == 0);
-    return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
     (void)dtd;
     assert(dtd.tableType == 1);
-    return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #else
-    return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
-                           HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+                           HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #endif
 }
 
-size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
 {
     /* validation checks */
     if (dstSize == 0) return ERROR(dstSize_tooSmall);
@@ -1732,160 +1869,14 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds
 #if defined(HUF_FORCE_DECOMPRESS_X1)
         (void)algoNb;
         assert(algoNb == 0);
-        return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+        return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
         (void)algoNb;
         assert(algoNb == 1);
-        return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+        return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
 #else
-        return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) :
-                        HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+        return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) :
+                        HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
 #endif
     }
 }
-
-#ifndef ZSTD_NO_UNUSED_FUNCTIONS
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_readDTableX1_wksp(DTable, src, srcSize,
-                                 workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
-                              const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
-                                       workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX);
-    return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize)
-{
-  U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-  return HUF_readDTableX2_wksp(DTable, src, srcSize,
-                               workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
-                              const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
-                                       workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
-    return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
-                                       workSpace, sizeof(workSpace));
-}
-size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX);
-    return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
-                              const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
-                                       workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
-    return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
-
-size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
-    static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 };
-#endif
-
-    /* validation checks */
-    if (dstSize == 0) return ERROR(dstSize_tooSmall);
-    if (cSrcSize > dstSize) return ERROR(corruption_detected);   /* invalid */
-    if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; }   /* not compressed */
-    if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; }   /* RLE */
-
-    {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-        (void)algoNb;
-        assert(algoNb == 0);
-        return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-        (void)algoNb;
-        assert(algoNb == 1);
-        return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize);
-#else
-        return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
-#endif
-    }
-}
-
-size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    /* validation checks */
-    if (dstSize == 0) return ERROR(dstSize_tooSmall);
-    if (cSrcSize > dstSize) return ERROR(corruption_detected);   /* invalid */
-    if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; }   /* not compressed */
-    if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; }   /* RLE */
-
-    {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-        (void)algoNb;
-        assert(algoNb == 0);
-        return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-        (void)algoNb;
-        assert(algoNb == 1);
-        return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
-#else
-        return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
-                        HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
-#endif
-    }
-}
-
-size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
-                                         workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
-                             const void* cSrc, size_t cSrcSize)
-{
-    U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
-    return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
-                                      workSpace, sizeof(workSpace));
-}
-#endif
diff --git a/lib/decompress/huf_decompress_amd64.S b/lib/decompress/huf_decompress_amd64.S
index 3f0e5c26c..671624fe3 100644
--- a/lib/decompress/huf_decompress_amd64.S
+++ b/lib/decompress/huf_decompress_amd64.S
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -30,14 +30,14 @@
  * TODO: Support Windows calling convention.
  */
 
-ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
-ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop)
-ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop)
-ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
-.global HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop
-.global HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop
-.global _HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop
-.global _HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop
+ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_fast_asm_loop)
+ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_fast_asm_loop)
+ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_fast_asm_loop)
+ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_fast_asm_loop)
+.global HUF_decompress4X1_usingDTable_internal_fast_asm_loop
+.global HUF_decompress4X2_usingDTable_internal_fast_asm_loop
+.global _HUF_decompress4X1_usingDTable_internal_fast_asm_loop
+.global _HUF_decompress4X2_usingDTable_internal_fast_asm_loop
 .text
 
 /* Sets up register mappings for clarity.
@@ -95,8 +95,9 @@ ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
 /* Define both _HUF_* & HUF_* symbols because MacOS
  * C symbols are prefixed with '_' & Linux symbols aren't.
  */
-_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
-HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
+_HUF_decompress4X1_usingDTable_internal_fast_asm_loop:
+HUF_decompress4X1_usingDTable_internal_fast_asm_loop:
+    ZSTD_CET_ENDBRANCH
     /* Save all registers - even if they are callee saved for simplicity. */
     push %rax
     push %rbx
@@ -350,8 +351,9 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
     pop %rax
     ret
 
-_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
-HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
+_HUF_decompress4X2_usingDTable_internal_fast_asm_loop:
+HUF_decompress4X2_usingDTable_internal_fast_asm_loop:
+    ZSTD_CET_ENDBRANCH
     /* Save all registers - even if they are callee saved for simplicity. */
     push %rax
     push %rbx
diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c
index 6ffa35f6e..ad5c34a7f 100644
--- a/lib/decompress/zstd_ddict.c
+++ b/lib/decompress/zstd_ddict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -19,7 +19,6 @@
 #include "../common/mem.h"         /* low level memory routines */
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "zstd_decompress_internal.h"
 #include "zstd_ddict.h"
diff --git a/lib/decompress/zstd_ddict.h b/lib/decompress/zstd_ddict.h
index bd03268b5..c4ca8877a 100644
--- a/lib/decompress/zstd_ddict.h
+++ b/lib/decompress/zstd_ddict.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 9fe326e73..d487966cc 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -59,7 +59,6 @@
 #include "../common/mem.h"         /* low level memory routines */
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
 #include "../common/zstd_internal.h"  /* blockProperties_t */
@@ -244,6 +243,7 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
     dctx->outBufferMode = ZSTD_bm_buffered;
     dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
     dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
+    dctx->disableHufAsm = 0;
 }
 
 static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
@@ -782,10 +782,11 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
             ip += 4;
         }
 
+        frameSizeInfo.nbBlocks = nbBlocks;
         frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
         frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
                                         ? zfh.frameContentSize
-                                        : nbBlocks * zfh.blockSizeMax;
+                                        : (unsigned long long)nbBlocks * zfh.blockSizeMax;
         return frameSizeInfo;
     }
 }
@@ -825,6 +826,48 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
     return bound;
 }
 
+size_t ZSTD_decompressionMargin(void const* src, size_t srcSize)
+{
+    size_t margin = 0;
+    unsigned maxBlockSize = 0;
+
+    /* Iterate over each frame */
+    while (srcSize > 0) {
+        ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+        size_t const compressedSize = frameSizeInfo.compressedSize;
+        unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
+        ZSTD_frameHeader zfh;
+
+        FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), "");
+        if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
+            return ERROR(corruption_detected);
+
+        if (zfh.frameType == ZSTD_frame) {
+            /* Add the frame header to our margin */
+            margin += zfh.headerSize;
+            /* Add the checksum to our margin */
+            margin += zfh.checksumFlag ? 4 : 0;
+            /* Add 3 bytes per block */
+            margin += 3 * frameSizeInfo.nbBlocks;
+
+            /* Compute the max block size */
+            maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax);
+        } else {
+            assert(zfh.frameType == ZSTD_skippableFrame);
+            /* Add the entire skippable frame size to our margin. */
+            margin += compressedSize;
+        }
+
+        assert(srcSize >= compressedSize);
+        src = (const BYTE*)src + compressedSize;
+        srcSize -= compressedSize;
+    }
+
+    /* Add the max block size back to the margin. */
+    margin += maxBlockSize;
+
+    return margin;
+}
 
 /*-*************************************************************
  *   Frame decoding
@@ -850,7 +893,7 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
         if (srcSize == 0) return 0;
         RETURN_ERROR(dstBuffer_null, "");
     }
-    ZSTD_memcpy(dst, src, srcSize);
+    ZSTD_memmove(dst, src, srcSize);
     return srcSize;
 }
 
@@ -928,6 +971,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
 
     /* Loop on each block */
     while (1) {
+        BYTE* oBlockEnd = oend;
         size_t decodedSize;
         blockProperties_t blockProperties;
         size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties);
@@ -937,16 +981,34 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
         remainingSrcSize -= ZSTD_blockHeaderSize;
         RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, "");
 
+        if (ip >= op && ip < oBlockEnd) {
+            /* We are decompressing in-place. Limit the output pointer so that we
+             * don't overwrite the block that we are currently reading. This will
+             * fail decompression if the input & output pointers aren't spaced
+             * far enough apart.
+             *
+             * This is important to set, even when the pointers are far enough
+             * apart, because ZSTD_decompressBlock_internal() can decide to store
+             * literals in the output buffer, after the block it is decompressing.
+             * Since we don't want anything to overwrite our input, we have to tell
+             * ZSTD_decompressBlock_internal to never write past ip.
+             *
+             * See ZSTD_allocateLiteralsBuffer() for reference.
+             */
+            oBlockEnd = op + (ip - op);
+        }
+
         switch(blockProperties.blockType)
         {
         case bt_compressed:
-            decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1, not_streaming);
+            decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, /* frame */ 1, not_streaming);
             break;
         case bt_raw :
+            /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */
             decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize);
             break;
         case bt_rle :
-            decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize);
+            decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize);
             break;
         case bt_reserved :
         default:
@@ -981,6 +1043,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
     }
     ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
     /* Allow caller to get size read */
+    DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr);
     *srcPtr = ip;
     *srcSizePtr = remainingSrcSize;
     return (size_t)(op-ostart);
@@ -1375,11 +1438,11 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
         /* in minimal huffman, we always use X1 variants */
         size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable,
                                                 dictPtr, dictEnd - dictPtr,
-                                                workspace, workspaceSize);
+                                                workspace, workspaceSize, /* flags */ 0);
 #else
         size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
                                                 dictPtr, (size_t)(dictEnd - dictPtr),
-                                                workspace, workspaceSize);
+                                                workspace, workspaceSize, /* flags */ 0);
 #endif
         RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
         dictPtr += hSize;
@@ -1549,7 +1612,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
  *  ZSTD_getFrameHeader(), which will provide a more precise error code. */
 unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
 {
-    ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 };
+    ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 };
     size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
     if (ZSTD_isError(hError)) return 0;
     return zfp.dictID;
@@ -1656,7 +1719,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
 size_t ZSTD_initDStream(ZSTD_DStream* zds)
 {
     DEBUGLOG(4, "ZSTD_initDStream");
-    return ZSTD_initDStream_usingDDict(zds, NULL);
+    FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), "");
+    FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), "");
+    return ZSTD_startingInputLength(zds->format);
 }
 
 /* ZSTD_initDStream_usingDDict() :
@@ -1664,6 +1729,7 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
  * this function cannot fail */
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
 {
+    DEBUGLOG(4, "ZSTD_initDStream_usingDDict");
     FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");
     FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");
     return ZSTD_startingInputLength(dctx->format);
@@ -1674,6 +1740,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
  * this function cannot fail */
 size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
 {
+    DEBUGLOG(4, "ZSTD_resetDStream");
     FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");
     return ZSTD_startingInputLength(dctx->format);
 }
@@ -1745,6 +1812,11 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
             bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
             bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
             return bounds;
+        case ZSTD_d_disableHuffmanAssembly:
+            bounds.lowerBound = 0;
+            bounds.upperBound = 1;
+            return bounds;
+
         default:;
     }
     bounds.error = ERROR(parameter_unsupported);
@@ -1785,6 +1857,9 @@ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value
         case ZSTD_d_refMultipleDDicts:
             *value = (int)dctx->refMultipleDDicts;
             return 0;
+        case ZSTD_d_disableHuffmanAssembly:
+            *value = (int)dctx->disableHufAsm;
+            return 0;
         default:;
     }
     RETURN_ERROR(parameter_unsupported, "");
@@ -1818,6 +1893,10 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
             }
             dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
             return 0;
+        case ZSTD_d_disableHuffmanAssembly:
+            CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value);
+            dctx->disableHufAsm = value != 0;
+            return 0;
         default:;
     }
     RETURN_ERROR(parameter_unsupported, "");
@@ -2058,6 +2137,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds));
                     if (ZSTD_isError(decompressedSize)) return decompressedSize;
                     DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
+                    assert(istart != NULL);
                     ip = istart + cSize;
                     op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */
                     zds->expected = 0;
@@ -2143,6 +2223,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 }
                 if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
                     FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), "");
+                    assert(ip != NULL);
                     ip += neededInSize;
                     /* Function modifies the stage so we must break */
                     break;
@@ -2157,7 +2238,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 int const isSkipFrame = ZSTD_isSkipFrame(zds);
                 size_t loadedSize;
                 /* At this point we shouldn't be decompressing a block that we can stream. */
-                assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip));
+                assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)));
                 if (isSkipFrame) {
                     loadedSize = MIN(toLoad, (size_t)(iend-ip));
                 } else {
@@ -2166,8 +2247,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                                     "should never happen");
                     loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip));
                 }
-                ip += loadedSize;
-                zds->inPos += loadedSize;
+                if (loadedSize != 0) {
+                    /* ip may be NULL */
+                    ip += loadedSize;
+                    zds->inPos += loadedSize;
+                }
                 if (loadedSize < toLoad) { someMoreWork = 0; break; }   /* not enough input, wait for more */
 
                 /* decode loaded input */
@@ -2214,8 +2298,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     if ((ip==istart) && (op==ostart)) {  /* no forward progress */
         zds->noForwardProgress ++;
         if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
-            RETURN_ERROR_IF(op==oend, dstSize_tooSmall, "");
-            RETURN_ERROR_IF(ip==iend, srcSize_wrong, "");
+            RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, "");
+            RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, "");
             assert(0);
         }
     } else {
@@ -2252,11 +2336,17 @@ size_t ZSTD_decompressStream_simpleArgs (
                             void* dst, size_t dstCapacity, size_t* dstPos,
                       const void* src, size_t srcSize, size_t* srcPos)
 {
-    ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
-    ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
-    /* ZSTD_compress_generic() will check validity of dstPos and srcPos */
-    size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
-    *dstPos = output.pos;
-    *srcPos = input.pos;
-    return cErr;
+    ZSTD_outBuffer output;
+    ZSTD_inBuffer  input;
+    output.dst = dst;
+    output.size = dstCapacity;
+    output.pos = *dstPos;
+    input.src = src;
+    input.size = srcSize;
+    input.pos = *srcPos;
+    {   size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
+        *dstPos = output.pos;
+        *srcPos = input.pos;
+        return cErr;
+    }
 }
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index e1ff21582..0a06a021e 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -20,7 +20,6 @@
 #include "../common/mem.h"         /* low level memory routines */
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "../common/zstd_internal.h"
 #include "zstd_decompress_internal.h"   /* ZSTD_DCtx */
@@ -142,6 +141,9 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 U32 const lhc = MEM_readLE32(istart);
                 size_t hufSuccess;
                 size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
+                int const flags = 0
+                    | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0)
+                    | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0);
                 switch(lhlCode)
                 {
                 case 0: case 1: default:   /* note : default is impossible, since lhlCode into [0..3] */
@@ -166,6 +168,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 }
                 RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
                 RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
+                if (!singleStream)
+                    RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong,
+                        "Not enough literals (%zu) for the 4-streams mode (min %u)",
+                        litSize, MIN_LITERALS_FOR_4_STREAMS);
                 RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
                 RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
                 ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
@@ -177,13 +183,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
 
                 if (litEncType==set_repeat) {
                     if (singleStream) {
-                        hufSuccess = HUF_decompress1X_usingDTable_bmi2(
+                        hufSuccess = HUF_decompress1X_usingDTable(
                             dctx->litBuffer, litSize, istart+lhSize, litCSize,
-                            dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
+                            dctx->HUFptr, flags);
                     } else {
-                        hufSuccess = HUF_decompress4X_usingDTable_bmi2(
+                        assert(litSize >= MIN_LITERALS_FOR_4_STREAMS);
+                        hufSuccess = HUF_decompress4X_usingDTable(
                             dctx->litBuffer, litSize, istart+lhSize, litCSize,
-                            dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
+                            dctx->HUFptr, flags);
                     }
                 } else {
                     if (singleStream) {
@@ -191,18 +198,18 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                         hufSuccess = HUF_decompress1X_DCtx_wksp(
                             dctx->entropy.hufTable, dctx->litBuffer, litSize,
                             istart+lhSize, litCSize, dctx->workspace,
-                            sizeof(dctx->workspace));
+                            sizeof(dctx->workspace), flags);
 #else
-                        hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2(
+                        hufSuccess = HUF_decompress1X1_DCtx_wksp(
                             dctx->entropy.hufTable, dctx->litBuffer, litSize,
                             istart+lhSize, litCSize, dctx->workspace,
-                            sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
+                            sizeof(dctx->workspace), flags);
 #endif
                     } else {
-                        hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2(
+                        hufSuccess = HUF_decompress4X_hufOnly_wksp(
                             dctx->entropy.hufTable, dctx->litBuffer, litSize,
                             istart+lhSize, litCSize, dctx->workspace,
-                            sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
+                            sizeof(dctx->workspace), flags);
                     }
                 }
                 if (dctx->litBufferLocation == ZSTD_split)
@@ -509,7 +516,8 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
                 for (i = 8; i < n; i += 8) {
                     MEM_write64(spread + pos + i, sv);
                 }
-                pos += n;
+                assert(n>=0);
+                pos += (size_t)n;
             }
         }
         /* Now we spread those positions across the table.
@@ -1162,7 +1170,7 @@ ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16
 }
 
 /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
- * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1)
+ * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32
  * bits before reloading. This value is the maximum number of bytes we read
  * after reloading when we are decoding long offsets.
  */
@@ -1212,6 +1220,10 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
         U32 const llnbBits = llDInfo->nbBits;
         U32 const mlnbBits = mlDInfo->nbBits;
         U32 const ofnbBits = ofDInfo->nbBits;
+
+        assert(llBits <= MaxLLBits);
+        assert(mlBits <= MaxMLBits);
+        assert(ofBits <= MaxOff);
         /*
          * As gcc has better branch and block analyzers, sometimes it is only
          * valuable to mark likeliness for clang, it gives around 3-4% of
@@ -1227,13 +1239,16 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
     #endif
                 ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
                 ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
-                assert(ofBits <= MaxOff);
+                ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32);
+                ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits);
                 if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) {
-                    U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed);
+                    /* Always read extra bits, this keeps the logic simple,
+                     * avoids branches, and avoids accidentally reading 0 bits.
+                     */
+                    U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32;
                     offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
                     BIT_reloadDStream(&seqState->DStream);
-                    if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
-                    assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32);   /* to avoid another reload */
+                    offset += BIT_readBitsFast(&seqState->DStream, extraBits);
                 } else {
                     offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
                     if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
@@ -1971,34 +1986,79 @@ ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
 #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
 
 
+/**
+ * @returns The total size of the history referencable by zstd, including
+ * both the prefix and the extDict. At @p op any offset larger than this
+ * is invalid.
+ */
+static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart)
+{
+    return (size_t)(op - virtualStart);
+}
 
-#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
-    !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
-/* ZSTD_getLongOffsetsShare() :
+typedef struct {
+    unsigned longOffsetShare;
+    unsigned maxNbAdditionalBits;
+} ZSTD_OffsetInfo;
+
+/* ZSTD_getOffsetInfo() :
  * condition : offTable must be valid
  * @return : "share" of long offsets (arbitrarily defined as > (1<<23))
- *           compared to maximum possible of (1< 22) total += 1;
+        assert(max <= (1 << OffFSELog));  /* max not too large */
+        for (u=0; u 22) info.longOffsetShare += 1;
+        }
+
+        assert(tableLog <= OffFSELog);
+        info.longOffsetShare <<= (OffFSELog - tableLog);  /* scale to OffFSELog */
     }
 
-    assert(tableLog <= OffFSELog);
-    total <<= (OffFSELog - tableLog);  /* scale to OffFSELog */
-
-    return total;
+    return info;
+}
+
+/**
+ * @returns The maximum offset we can decode in one read of our bitstream, without
+ * reloading more bits in the middle of the offset bits read. Any offsets larger
+ * than this must use the long offset decoder.
+ */
+static size_t ZSTD_maxShortOffset(void)
+{
+    if (MEM_64bits()) {
+        /* We can decode any offset without reloading bits.
+         * This might change if the max window size grows.
+         */
+        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+        return (size_t)-1;
+    } else {
+        /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1.
+         * This offBase would require STREAM_ACCUMULATOR_MIN extra bits.
+         * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset.
+         */
+        size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1;
+        size_t const maxOffset = maxOffbase - ZSTD_REP_NUM;
+        assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN);
+        return maxOffset;
+    }
 }
-#endif
 
 size_t
 ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
@@ -2006,16 +2066,17 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
                         const void* src, size_t srcSize, const int frame, const streaming_operation streaming)
 {   /* blockType == blockCompressed */
     const BYTE* ip = (const BYTE*)src;
-    /* isLongOffset must be true if there are long offsets.
-     * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN.
-     * We don't expect that to be the case in 64-bit mode.
-     * In block mode, window size is not known, so we have to be conservative.
-     * (note: but it could be evaluated from current-lowLimit)
-     */
-    ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))));
     DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
 
-    RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
+    /* Note : the wording of the specification
+     * allows compressed block to be sized exactly ZSTD_BLOCKSIZE_MAX.
+     * This generally does not happen, as it makes little sense,
+     * since an uncompressed block would feature same size and have no decompression cost.
+     * Also, note that decoder from reference libzstd before < v1.5.4
+     * would consider this edge case as an error.
+     * As a consequence, avoid generating compressed blocks of size ZSTD_BLOCKSIZE_MAX
+     * for broader compatibility with the deployed ecosystem of zstd decoders */
+    RETURN_ERROR_IF(srcSize > ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
 
     /* Decode literals section */
     {   size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
@@ -2027,6 +2088,23 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 
     /* Build Decoding Tables */
     {
+        /* Compute the maximum block size, which must also work when !frame and fParams are unset.
+         * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t.
+         */
+        size_t const blockSizeMax = MIN(dstCapacity, (frame ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX));
+        size_t const totalHistorySize = ZSTD_totalHistorySize((BYTE*)dst + blockSizeMax, (BYTE const*)dctx->virtualStart);
+        /* isLongOffset must be true if there are long offsets.
+         * Offsets are long if they are larger than ZSTD_maxShortOffset().
+         * We don't expect that to be the case in 64-bit mode.
+         *
+         * We check here to see if our history is large enough to allow long offsets.
+         * If it isn't, then we can't possible have (valid) long offsets. If the offset
+         * is invalid, then it is okay to read it incorrectly.
+         *
+         * If isLongOffsets is true, then we will later check our decoding table to see
+         * if it is even possible to generate long offsets.
+         */
+        ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset()));
         /* These macros control at build-time which decompressor implementation
          * we use. If neither is defined, we do some inspection and dispatch at
          * runtime.
@@ -2034,6 +2112,11 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
     !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
         int usePrefetchDecoder = dctx->ddictIsCold;
+#else
+        /* Set to 1 to avoid computing offset info if we don't need to.
+         * Otherwise this value is ignored.
+         */
+        int usePrefetchDecoder = 1;
 #endif
         int nbSeq;
         size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize);
@@ -2043,26 +2126,38 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 
         RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
 
-#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
-    !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
-        if ( !usePrefetchDecoder
-          && (!frame || (dctx->fParams.windowSize > (1<<24)))
-          && (nbSeq>ADVANCED_SEQS) ) {  /* could probably use a larger nbSeq limit */
-            U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr);
-            U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
-            usePrefetchDecoder = (shareLongOffsets >= minShare);
+        /* If we could potentially have long offsets, or we might want to use the prefetch decoder,
+         * compute information about the share of long offsets, and the maximum nbAdditionalBits.
+         * NOTE: could probably use a larger nbSeq limit
+         */
+        if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) {
+            ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq);
+            if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) {
+                /* If isLongOffset, but the maximum number of additional bits that we see in our table is small
+                 * enough, then we know it is impossible to have too long an offset in this block, so we can
+                 * use the regular offset decoder.
+                 */
+                isLongOffset = ZSTD_lo_isRegularOffset;
+            }
+            if (!usePrefetchDecoder) {
+                U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
+                usePrefetchDecoder = (info.longOffsetShare >= minShare);
+            }
         }
-#endif
 
         dctx->ddictIsCold = 0;
 
 #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
     !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
-        if (usePrefetchDecoder)
+        if (usePrefetchDecoder) {
+#else
+        (void)usePrefetchDecoder;
+        {
 #endif
 #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
             return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
 #endif
+        }
 
 #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
         /* else */
diff --git a/lib/decompress/zstd_decompress_block.h b/lib/decompress/zstd_decompress_block.h
index c61a9d0c4..67791dbc3 100644
--- a/lib/decompress/zstd_decompress_block.h
+++ b/lib/decompress/zstd_decompress_block.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h
index 91e9dceb5..c2ec5d9fb 100644
--- a/lib/decompress/zstd_decompress_internal.h
+++ b/lib/decompress/zstd_decompress_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -165,6 +165,7 @@ struct ZSTD_DCtx_s
     ZSTD_dictUses_e dictUses;
     ZSTD_DDictHashSet* ddictSet;                    /* Hash set for multiple ddicts */
     ZSTD_refMultipleDDicts_e refMultipleDDicts;     /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
+    int disableHufAsm;
 
     /* streaming */
     ZSTD_dStreamStage streamStage;
diff --git a/lib/deprecated/zbuff.h b/lib/deprecated/zbuff.h
index b83ea0fed..a968245b3 100644
--- a/lib/deprecated/zbuff.h
+++ b/lib/deprecated/zbuff.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_common.c b/lib/deprecated/zbuff_common.c
index e7d01a081..5a2f2db35 100644
--- a/lib/deprecated/zbuff_common.c
+++ b/lib/deprecated/zbuff_common.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_compress.c b/lib/deprecated/zbuff_compress.c
index 51cf158c4..1d8682150 100644
--- a/lib/deprecated/zbuff_compress.c
+++ b/lib/deprecated/zbuff_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/deprecated/zbuff_decompress.c b/lib/deprecated/zbuff_decompress.c
index d73c0f35f..12a66af74 100644
--- a/lib/deprecated/zbuff_decompress.c
+++ b/lib/deprecated/zbuff_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -13,6 +13,8 @@
 /* *************************************
 *  Dependencies
 ***************************************/
+#define ZSTD_DISABLE_DEPRECATE_WARNINGS  /* suppress warning on ZSTD_initDStream_usingDict */
+#include "../zstd.h"        /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
 #define ZBUFF_STATIC_LINKING_ONLY
 #include "zbuff.h"
 
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index 724675d30..9e5e7d5b5 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -647,7 +647,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
 
 void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel)
 {
-  const double ratio = (double)nbDmers / maxDictSize;
+  const double ratio = (double)nbDmers / (double)maxDictSize;
   if (ratio >= 10) {
       return;
   }
@@ -951,9 +951,17 @@ void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters,
   }
 }
 
+static COVER_dictSelection_t setDictSelection(BYTE* buf, size_t s, size_t csz)
+{
+    COVER_dictSelection_t ds;
+    ds.dictContent = buf;
+    ds.dictSize = s;
+    ds.totalCompressedSize = csz;
+    return ds;
+}
+
 COVER_dictSelection_t COVER_dictSelectionError(size_t error) {
-    COVER_dictSelection_t selection = { NULL, 0, error };
-    return selection;
+    return setDictSelection(NULL, 0, error);
 }
 
 unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection) {
@@ -1006,9 +1014,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
   }
 
   if (params.shrinkDict == 0) {
-    COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize };
     free(candidateDictBuffer);
-    return selection;
+    return setDictSelection(largestDictbuffer, dictContentSize, totalCompressedSize);
   }
 
   largestDict = dictContentSize;
@@ -1040,20 +1047,16 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
       return COVER_dictSelectionError(totalCompressedSize);
     }
 
-    if (totalCompressedSize <= largestCompressed * regressionTolerance) {
-      COVER_dictSelection_t selection = { candidateDictBuffer, dictContentSize, totalCompressedSize };
+    if ((double)totalCompressedSize <= (double)largestCompressed * regressionTolerance) {
       free(largestDictbuffer);
-      return selection;
+      return setDictSelection( candidateDictBuffer, dictContentSize, totalCompressedSize );
     }
     dictContentSize *= 2;
   }
   dictContentSize = largestDict;
   totalCompressedSize = largestCompressed;
-  {
-    COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize };
-    free(candidateDictBuffer);
-    return selection;
-  }
+  free(candidateDictBuffer);
+  return setDictSelection( largestDictbuffer, dictContentSize, totalCompressedSize );
 }
 
 /**
diff --git a/lib/dictBuilder/cover.h b/lib/dictBuilder/cover.h
index 1aacdddd6..252624bde 100644
--- a/lib/dictBuilder/cover.h
+++ b/lib/dictBuilder/cover.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c
index 63a2cee72..46bba0120 100644
--- a/lib/dictBuilder/fastcover.c
+++ b/lib/dictBuilder/fastcover.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index a932276ff..f22e04df2 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -44,7 +44,6 @@
 #ifndef ZDICT_STATIC_LINKING_ONLY
 #  define ZDICT_STATIC_LINKING_ONLY
 #endif
-#define HUF_STATIC_LINKING_ONLY
 
 #include "../common/mem.h"           /* read */
 #include "../common/fse.h"           /* FSE_normalizeCount, FSE_writeNCount */
@@ -373,7 +372,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const
             elt = table[u];
             /* sort : improve rank */
             while ((u>1) && (table[u-1].savings < elt.savings))
-            table[u] = table[u-1], u--;
+                table[u] = table[u-1], u--;
             table[u] = elt;
             return u;
     }   }
@@ -524,7 +523,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
             if (solution.length==0) { cursor++; continue; }
             ZDICT_insertDictItem(dictList, dictListSize, solution, buffer);
             cursor += solution.length;
-            DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100);
+            DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / (double)bufferSize * 100.0);
     }   }
 
 _cleanup:
@@ -676,6 +675,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles);
     size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles);
     BYTE* dstPtr = (BYTE*)dstBuffer;
+    U32 wksp[HUF_CTABLE_WORKSPACE_SIZE_U32];
 
     /* init */
     DEBUGLOG(4, "ZDICT_analyzeEntropy");
@@ -716,7 +716,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     }   }
 
     /* analyze, build stats, starting with literals */
-    {   size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
+    {   size_t maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp));
         if (HUF_isError(maxNbBits)) {
             eSize = maxNbBits;
             DISPLAYLEVEL(1, " HUF_buildCTable error \n");
@@ -725,7 +725,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
         if (maxNbBits==8) {  /* not compressible : will fail on HUF_writeCTable() */
             DISPLAYLEVEL(2, "warning : pathological dataset : literals are not compressible : samples are noisy or too regular \n");
             ZDICT_flatLit(countLit);  /* replace distribution by a fake "mostly flat but still compressible" distribution, that HUF_writeCTable() can encode */
-            maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
+            maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp));
             assert(maxNbBits==9);
         }
         huffLog = (U32)maxNbBits;
@@ -766,7 +766,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     llLog = (U32)errorCode;
 
     /* write result to buffer */
-    {   size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
+    {   size_t const hhSize = HUF_writeCTable_wksp(dstPtr, maxDstSize, hufTable, 255, huffLog, wksp, sizeof(wksp));
         if (HUF_isError(hhSize)) {
             eSize = hhSize;
             DISPLAYLEVEL(1, "HUF_writeCTable error \n");
diff --git a/lib/dll/example/Makefile b/lib/dll/example/Makefile
index 03b034dd5..86cf6906e 100644
--- a/lib/dll/example/Makefile
+++ b/lib/dll/example/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index a6f1174b8..dd173251d 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -242,6 +242,13 @@ MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size
         frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
         frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
     }
+    /* In all cases, decompressedBound == nbBlocks * ZSTD_BLOCKSIZE_MAX.
+     * So we can compute nbBlocks without having to change every function.
+     */
+    if (frameSizeInfo.decompressedBound != ZSTD_CONTENTSIZE_ERROR) {
+        assert((frameSizeInfo.decompressedBound & (ZSTD_BLOCKSIZE_MAX - 1)) == 0);
+        frameSizeInfo.nbBlocks = (size_t)(frameSizeInfo.decompressedBound / ZSTD_BLOCKSIZE_MAX);
+    }
     return frameSizeInfo;
 }
 
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index 23caaef56..1a3aad07e 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -190,25 +190,6 @@ typedef   signed long long  S64;
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* FSE_FORCE_MEMORY_ACCESS
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets generating assembly depending on alignment.
- *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
- */
-#ifndef FSE_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
-#    define FSE_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
-
 
 static unsigned FSE_32bits(void)
 {
@@ -221,24 +202,6 @@ static unsigned FSE_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==2)
-
-static U16 FSE_read16(const void* memPtr) { return *(const U16*) memPtr; }
-static U32 FSE_read32(const void* memPtr) { return *(const U32*) memPtr; }
-static U64 FSE_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-#elif defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==1)
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
-
-static U16 FSE_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-static U32 FSE_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-static U64 FSE_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
-
-#else
-
 static U16 FSE_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -254,8 +217,6 @@ static U64 FSE_read64(const void* memPtr)
     U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
 }
 
-#endif /* FSE_FORCE_MEMORY_ACCESS */
-
 static U16 FSE_readLE16(const void* memPtr)
 {
     if (FSE_isLittleEndian())
@@ -1190,7 +1151,7 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
     zstd - standard compression library
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -1759,20 +1720,26 @@ static size_t ZSTD_execSequence(BYTE* op,
     static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};   /* added */
     static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11};   /* subtracted */
     const BYTE* const ostart = op;
+    BYTE* const oLitEnd = op + sequence.litLength;
     const size_t litLength = sequence.litLength;
     BYTE* const endMatch = op + litLength + sequence.matchLength;    /* risk : address space overflow (32-bits) */
     const BYTE* const litEnd = *litPtr + litLength;
 
-    /* check */
+    /* checks */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
+    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
+
     if (endMatch > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
-    if (litEnd > litLimit) return ERROR(corruption_detected);
-    if (sequence.matchLength > (size_t)(*litPtr-op))  return ERROR(dstSize_tooSmall);    /* overwrite literal segment */
+    if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
+    if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall);  /* overwrite literal segment */
 
     /* copy Literals */
-    if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8))
-        memmove(op, *litPtr, litLength);   /* overwrite risk */
-    else
-        ZSTD_wildcopy(op, *litPtr, litLength);
+    ZSTD_memmove(op, *litPtr, sequence.litLength);   /* note : v0.1 seems to allow scenarios where output or input are close to end of buffer */
+
     op += litLength;
     *litPtr = litEnd;   /* update for next sequence */
 
diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h
index f777eb6e4..6ac876954 100644
--- a/lib/legacy/zstd_v01.h
+++ b/lib/legacy/zstd_v01.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index 6eadd3948..e09bb4a24 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -28,7 +28,7 @@
    low-level memory access routines
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -115,24 +115,6 @@ extern "C" {
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets generating assembly depending on alignment.
- *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
@@ -143,33 +125,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -190,9 +145,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
-
 MEM_STATIC U16 MEM_readLE16(const void* memPtr)
 {
     if (MEM_isLittleEndian())
@@ -269,7 +221,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
    header file (to include)
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -510,7 +462,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
    Error codes and messages
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -609,7 +561,7 @@ typedef unsigned FSE_DTable;   /* don't allocate that. It's just a way to be mor
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -753,7 +705,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -822,7 +774,7 @@ static size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, si
     Header File
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -882,7 +834,7 @@ typedef struct ZSTD_CCtx_s ZSTD_CCtx;   /* incomplete type */
     Header File for static linking only
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -946,7 +898,7 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
    FSE : Finite State Entropy coder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1450,7 +1402,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
    Huff0 : Huffman coder, part of New Generation Entropy library
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2609,7 +2561,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
     zstd - standard compression library
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -3114,12 +3066,19 @@ static size_t ZSTD_execSequence(BYTE* op,
     const BYTE* const litEnd = *litPtr + sequence.litLength;
 
     /* checks */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use the pointer check */
+    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
+    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
+
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
     if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
 
     /* copy Literals */
-    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = litEnd;   /* update for next sequence */
 
diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h
index 1b371953b..dab0260ee 100644
--- a/lib/legacy/zstd_v02.h
+++ b/lib/legacy/zstd_v02.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 72fc9140d..b0d7f521e 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -29,7 +29,7 @@
    low-level memory access routines
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -116,24 +116,6 @@ extern "C" {
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets generating assembly depending on alignment.
- *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
@@ -144,33 +126,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -191,10 +146,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
-
 MEM_STATIC U16 MEM_readLE16(const void* memPtr)
 {
     if (MEM_isLittleEndian())
@@ -271,7 +222,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
    header file (to include)
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -512,7 +463,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
    Error codes and messages
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -611,7 +562,7 @@ typedef unsigned FSE_DTable;   /* don't allocate that. It's just a way to be mor
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -755,7 +706,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -823,7 +774,7 @@ static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, si
     Header File
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -883,7 +834,7 @@ typedef struct ZSTD_CCtx_s ZSTD_CCtx;   /* incomplete type */
     Header File for static linking only
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -947,7 +898,7 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
    FSE : Finite State Entropy coder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1451,7 +1402,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
    Huff0 : Huffman coder, part of New Generation Entropy library
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2248,7 +2199,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
     zstd - standard compression library
     Copyright (C) 2014-2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2755,18 +2706,24 @@ static size_t ZSTD_execSequence(BYTE* op,
     const BYTE* const litEnd = *litPtr + sequence.litLength;
 
     /* checks */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
+    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
+    if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
+
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
     if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
 
     /* copy Literals */
-    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = litEnd;   /* update for next sequence */
 
     /* copy Match */
-    {
-        const BYTE* match = op - sequence.offset;
+    {   const BYTE* match = op - sequence.offset;
 
         /* check */
         if (sequence.offset > (size_t)op) return ERROR(corruption_detected);   /* address space overflow test (this test seems kept by clang optimizer) */
diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h
index 7a00d4304..9bf3cce64 100644
--- a/lib/legacy/zstd_v03.h
+++ b/lib/legacy/zstd_v03.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index a2d281eb9..57be832bd 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -87,24 +87,6 @@ extern "C" {
 /****************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets generating assembly depending on alignment.
- *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
@@ -115,33 +97,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -162,9 +117,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
-
 MEM_STATIC U16 MEM_readLE16(const void* memPtr)
 {
     if (MEM_isLittleEndian())
@@ -542,7 +494,7 @@ If there is an error, the function will return an error code, which can be teste
    header file (to include)
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -781,7 +733,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -930,7 +882,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
    FSE : Finite State Entropy coder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1436,7 +1388,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
    header file
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1514,7 +1466,7 @@ static unsigned    HUF_isError(size_t code);        /* tells if a return value i
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1601,7 +1553,7 @@ static size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const
    Huff0 : Huffman coder, part of New Generation Entropy library
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2401,7 +2353,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
     zstd - decompression module fo v0.4 legacy format
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2876,13 +2828,19 @@ static size_t ZSTD_execSequence(BYTE* op,
     const BYTE* const litEnd = *litPtr + sequence.litLength;
     const BYTE* match = oLitEnd - sequence.offset;
 
-    /* check */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    /* checks */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
+    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
+
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
-    if (litEnd > litLimit) return ERROR(corruption_detected);   /* risk read beyond lit buffer */
+    if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
 
     /* copy Literals */
-    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = litEnd;   /* update for next sequence */
 
@@ -3283,7 +3241,7 @@ static void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* dict, s
     Buffered version of Zstd compression library
     Copyright (C) 2015, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h
index 66b97ab8e..640240d62 100644
--- a/lib/legacy/zstd_v04.h
+++ b/lib/legacy/zstd_v04.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 216c67d01..93a1169f3 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -19,7 +19,7 @@
    low-level memory access routines
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -106,24 +106,6 @@ extern "C" {
 /*-**************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets depending on alignment.
- *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
@@ -134,37 +116,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard, by lying on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -195,9 +146,6 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
-
 MEM_STATIC U16 MEM_readLE16(const void* memPtr)
 {
     if (MEM_isLittleEndian())
@@ -262,7 +210,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
     Header File for static linking only
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -286,7 +234,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net
+    - zstd homepage : https://facebook.github.io/zstd
 */
 #ifndef ZSTD_STATIC_H
 #define ZSTD_STATIC_H
@@ -398,7 +346,7 @@ size_t ZSTDv05_decompressBlock(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity
     Header File for include
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -553,7 +501,7 @@ typedef struct {
    header file
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -672,7 +620,7 @@ size_t FSEv05_decompress_usingDTable(void* dst, size_t dstCapacity, const void*
    header file (to include)
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -901,7 +849,7 @@ MEM_STATIC unsigned BITv05_endOfDStream(const BITv05_DStream_t* DStream)
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1051,7 +999,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
    FSEv05 : Finite State Entropy coder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1537,7 +1485,7 @@ size_t FSEv05_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t
    header file
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1610,7 +1558,7 @@ const char* HUFv05_getErrorName(size_t code);   /* provides error code string (u
    header file, for static linking only
    Copyright (C) 2013-2016, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1702,7 +1650,7 @@ size_t HUFv05_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void
    Huff0 : Huffman coder, part of New Generation Entropy library
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2547,7 +2495,7 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
     zstd - standard compression library
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -3234,13 +3182,19 @@ static size_t ZSTDv05_execSequence(BYTE* op,
     const BYTE* const litEnd = *litPtr + sequence.litLength;
     const BYTE* match = oLitEnd - sequence.offset;
 
-    /* check */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    /* checks */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
+    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
+
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
-    if (litEnd > litLimit) return ERROR(corruption_detected);   /* risk read beyond lit buffer */
+    if (litEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
 
     /* copy Literals */
-    ZSTDv05_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTDv05_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = litEnd;   /* update for next sequence */
 
@@ -3746,7 +3700,7 @@ size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, s
     Buffered version of Zstd compression library
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h
index bd423bfc1..2dcffc923 100644
--- a/lib/legacy/zstd_v05.h
+++ b/lib/legacy/zstd_v05.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index b224355d4..175f7cc42 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -23,7 +23,7 @@
    low-level memory access routines
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -108,24 +108,6 @@ extern "C" {
 /*-**************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets depending on alignment.
- *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
@@ -136,33 +118,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard, by lying on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -183,9 +138,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
 MEM_STATIC U32 MEM_swap32(U32 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */
@@ -281,7 +233,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
     Header File for static linking only
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -305,7 +257,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net
+    - zstd homepage : https://facebook.github.io/zstd
 */
 #ifndef ZSTDv06_STATIC_H
 #define ZSTDv06_STATIC_H
@@ -412,7 +364,7 @@ ZSTDLIBv06_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, siz
     Header File for include
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -619,7 +571,7 @@ void ZSTDv06_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
    Public Prototypes declaration
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -767,7 +719,7 @@ If there is an error, the function will return an error code, which can be teste
    header file (to include)
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1002,7 +954,7 @@ MEM_STATIC unsigned BITv06_endOfDStream(const BITv06_DStream_t* DStream)
    header file for static linking (only)
    Copyright (C) 2013-2015, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1210,7 +1162,7 @@ MEM_STATIC BYTE FSEv06_decodeSymbolFast(FSEv06_DState_t* DStatePtr, BITv06_DStre
    Common functions of New Generation Entropy library
    Copyright (C) 2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1355,7 +1307,7 @@ size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned
    FSE : Finite State Entropy decoder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1679,7 +1631,7 @@ size_t FSEv06_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t
    header file
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1749,7 +1701,7 @@ size_t HUFv06_compressBound(size_t size);       /**< maximum compressed size */
    header file, for static linking only
    Copyright (C) 2013-2016, Yann Collet
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1931,7 +1883,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
    Huffman decoder, part of New Generation Entropy library
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2676,7 +2628,7 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
     Common functions of Zstd compression library
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2700,7 +2652,7 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
+    - zstd homepage : https://facebook.github.io/zstd/
 */
 
 
@@ -2730,7 +2682,7 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er
     zstd - standard compression library
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2754,7 +2706,7 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net
+    - zstd homepage : https://facebook.github.io/zstd
 */
 
 /* ***************************************************************
@@ -3370,13 +3322,19 @@ static size_t ZSTDv06_execSequence(BYTE* op,
     const BYTE* const iLitEnd = *litPtr + sequence.litLength;
     const BYTE* match = oLitEnd - sequence.offset;
 
-    /* check */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    /* checks */
+    size_t const seqLength = sequence.litLength + sequence.matchLength;
+
+    if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
+    /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
+    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
+
     if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
-    if (iLitEnd > litLimit) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
+    if (iLitEnd > litLimit) return ERROR(corruption_detected);   /* overRead beyond lit buffer */
 
     /* copy Literals */
-    ZSTDv06_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTDv06_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = iLitEnd;   /* update for next sequence */
 
@@ -3889,7 +3847,7 @@ size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, s
     Buffered version of Zstd compression library
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -3913,7 +3871,7 @@ size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, s
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
+    - zstd homepage : https://facebook.github.io/zstd/
 */
 
 
@@ -4035,7 +3993,8 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
                     size_t const toLoad = hSize - zbd->lhSize;   /* if hSize!=0, hSize > zbd->lhSize */
                     if (ZSTDv06_isError(hSize)) return hSize;
                     if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
-                        memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
+                        if (ip != NULL)
+                            memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
                         zbd->lhSize += iend-ip;
                         *dstCapacityPtr = 0;
                         return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize;   /* remaining header bytes + next block header */
diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h
index 9e32b76e0..633891010 100644
--- a/lib/legacy/zstd_v06.h
+++ b/lib/legacy/zstd_v06.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index c12a091cb..15dc3ef79 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -184,7 +184,7 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
    low-level memory access routines
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -268,24 +268,6 @@ extern "C" {
 /*-**************************************************************
 *  Memory I/O
 *****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
- * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets depending on alignment.
- *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
-#  if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
-#    define MEM_FORCE_MEMORY_ACCESS 1
-#  endif
-#endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
@@ -296,33 +278,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
     return one.c[0];
 }
 
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard, by lying on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-
-#else
-
-/* default method, safe and standard.
-   can sometimes prove slower */
-
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -343,8 +298,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
     memcpy(memPtr, &value, sizeof(value));
 }
 
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
 MEM_STATIC U32 MEM_swap32(U32 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */
@@ -439,7 +392,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
    header file (to include)
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -670,7 +623,7 @@ MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* DStream)
    Public Prototypes declaration
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -978,7 +931,7 @@ MEM_STATIC BYTE FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_DStre
    header file
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1151,7 +1104,7 @@ size_t HUFv07_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void
    Common functions of New Generation Entropy library
    Copyright (C) 2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1375,7 +1328,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
    FSE : Finite State Entropy decoder
    Copyright (C) 2013-2015, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -1699,7 +1652,7 @@ size_t FSEv07_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t
    Huffman decoder, part of New Generation Entropy library
    Copyright (C) 2013-2016, Yann Collet.
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
@@ -2577,7 +2530,7 @@ size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize,
     Common functions of Zstd compression library
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2601,7 +2554,7 @@ size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize,
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
+    - zstd homepage : https://facebook.github.io/zstd/
 */
 
 
@@ -2647,7 +2600,7 @@ static void ZSTDv07_defaultFreeFunction(void* opaque, void* address)
     Header File for include
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2854,7 +2807,7 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction
     zstd - standard compression library
     Copyright (C) 2014-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -2878,7 +2831,7 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net
+    - zstd homepage : https://facebook.github.io/zstd
 */
 
 /* ***************************************************************
@@ -3599,11 +3552,14 @@ size_t ZSTDv07_execSequence(BYTE* op,
     const BYTE* match = oLitEnd - sequence.offset;
 
     /* check */
-    if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
-    if (iLitEnd > litLimit) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
+    assert(oend >= op);
+    if (sequence.litLength + WILDCOPY_OVERLENGTH > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    if (sequenceLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
+    assert(litLimit >= *litPtr);
+    if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);;
 
     /* copy Literals */
-    ZSTDv07_wildcopy(op, *litPtr, sequence.litLength);   /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
+    ZSTDv07_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength);   /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = iLitEnd;   /* update for next sequence */
 
@@ -3617,7 +3573,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
             return sequenceLength;
         }
         /* span extDict & currentPrefixSegment */
-        {   size_t const length1 = dictEnd - match;
+        {   size_t const length1 = (size_t)(dictEnd - match);
             memmove(oLitEnd, match, length1);
             op = oLitEnd + length1;
             sequence.matchLength -= length1;
@@ -4253,7 +4209,7 @@ ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
     Buffered version of Zstd compression library
     Copyright (C) 2015-2016, Yann Collet.
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+    BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
@@ -4277,7 +4233,7 @@ ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
+    - zstd homepage : https://facebook.github.io/zstd/
 */
 
 
@@ -4417,7 +4373,8 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
                 if (hSize != 0) {
                     size_t const toLoad = hSize - zbd->lhSize;   /* if hSize!=0, hSize > zbd->lhSize */
                     if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
-                        memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
+                        if (ip != NULL)
+                            memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
                         zbd->lhSize += iend-ip;
                         *dstCapacityPtr = 0;
                         return (hSize - zbd->lhSize) + ZSTDv07_blockHeaderSize;   /* remaining header bytes + next block header */
diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h
index bc35cfa6a..1ff39041f 100644
--- a/lib/legacy/zstd_v07.h
+++ b/lib/legacy/zstd_v07.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/libzstd.mk b/lib/libzstd.mk
index c97db5628..5e11d5d29 100644
--- a/lib/libzstd.mk
+++ b/lib/libzstd.mk
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -15,17 +15,34 @@
 # Zstd lib directory
 LIBZSTD ?= ./
 
+# ZSTD_LIB_MINIFY is a helper variable that
+# configures a bunch of other variables to space-optimized defaults.
+ZSTD_LIB_MINIFY ?= 0
+
 # Legacy support
-ZSTD_LEGACY_SUPPORT ?= 5
+ifneq ($(ZSTD_LIB_MINIFY), 0)
+  ZSTD_LEGACY_SUPPORT ?= 0
+else
+  ZSTD_LEGACY_SUPPORT ?= 5
+endif
 ZSTD_LEGACY_MULTITHREADED_API ?= 0
 
 # Build size optimizations
-HUF_FORCE_DECOMPRESS_X1 ?= 0
-HUF_FORCE_DECOMPRESS_X2 ?= 0
-ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 0
-ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0
-ZSTD_NO_INLINE ?= 0
-ZSTD_STRIP_ERROR_STRINGS ?= 0
+ifneq ($(ZSTD_LIB_MINIFY), 0)
+  HUF_FORCE_DECOMPRESS_X1 ?= 1
+  HUF_FORCE_DECOMPRESS_X2 ?= 0
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 1
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0
+  ZSTD_NO_INLINE ?= 1
+  ZSTD_STRIP_ERROR_STRINGS ?= 1
+else
+  HUF_FORCE_DECOMPRESS_X1 ?= 0
+  HUF_FORCE_DECOMPRESS_X2 ?= 0
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 0
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0
+  ZSTD_NO_INLINE ?= 0
+  ZSTD_STRIP_ERROR_STRINGS ?= 0
+endif
 
 # Assembly support
 ZSTD_NO_ASM ?= 0
@@ -61,17 +78,8 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
 CCVER := $(shell $(CC) --version)
 ZSTD_VERSION?= $(LIBVER)
 
-# ZSTD_LIB_MINIFY is a helper variable that
-# configures a bunch of other variables to space-optimized defaults.
-ZSTD_LIB_MINIFY ?= 0
 ifneq ($(ZSTD_LIB_MINIFY), 0)
   HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
-  ZSTD_LEGACY_SUPPORT ?= 0
-  ZSTD_LIB_DEPRECATED ?= 0
-  HUF_FORCE_DECOMPRESS_X1 ?= 1
-  ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 1
-  ZSTD_NO_INLINE ?= 1
-  ZSTD_STRIP_ERROR_STRINGS ?= 1
 ifneq ($(HAVE_CC_OZ), 0)
     # Some compilers (clang) support an even more space-optimized setting.
     CFLAGS += -Oz
diff --git a/lib/libzstd.pc.in b/lib/libzstd.pc.in
index 43ebaec35..d5cc0270c 100644
--- a/lib/libzstd.pc.in
+++ b/lib/libzstd.pc.in
@@ -1,6 +1,6 @@
 #   ZSTD - standard compression algorithm
-#   Copyright (C) 2014-2016, Yann Collet, Facebook
-#   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+#   Copyright (c) Meta Platforms, Inc. and affiliates.
+#   BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
 
 prefix=@PREFIX@
 exec_prefix=@EXEC_PREFIX@
@@ -9,7 +9,7 @@ libdir=@LIBDIR@
 
 Name: zstd
 Description: fast lossless compression algorithm library
-URL: http://www.zstd.net/
+URL: https://facebook.github.io/zstd/
 Version: @VERSION@
 Libs: -L${libdir} -lzstd
 Libs.private: @LIBS_PRIVATE@
diff --git a/lib/module.modulemap b/lib/module.modulemap
index bbb939782..eff98dfac 100644
--- a/lib/module.modulemap
+++ b/lib/module.modulemap
@@ -1,17 +1,27 @@
 module libzstd [extern_c] {
     header "zstd.h"
     export *
-    config_macros [exhaustive] /* zstd.h */ \
+    config_macros [exhaustive] \
+        /* zstd.h */ \
         ZSTD_STATIC_LINKING_ONLY, \
+        ZSTDLIB_VISIBILITY, \
         ZSTDLIB_VISIBLE, \
+        ZSTDLIB_HIDDEN, \
         ZSTD_DLL_EXPORT, \
         ZSTDLIB_STATIC_API, \
         ZSTD_DISABLE_DEPRECATE_WARNINGS, \
         ZSTD_CLEVEL_DEFAULT, \
-        /* zdict.h */ ZDICT_STATIC_LINKING_ONLY, \
+        /* zdict.h */ \
+        ZDICT_STATIC_LINKING_ONLY, \
+        ZDICTLIB_VISIBLE, \
+        ZDICTLIB_HIDDEN, \
         ZDICTLIB_VISIBILITY, \
+        ZDICTLIB_STATIC_API, \
         ZDICT_DISABLE_DEPRECATE_WARNINGS, \
-        /* zstd_errors.h */ ZSTDERRORLIB_VISIBILITY
+        /* zstd_errors.h */ \
+        ZSTDERRORLIB_VISIBLE, \
+        ZSTDERRORLIB_HIDDEN, \
+        ZSTDERRORLIB_VISIBILITY
 
     module dictbuilder [extern_c] {
         header "zdict.h"
diff --git a/lib/zdict.h b/lib/zdict.h
index 8e21ba0f7..2268f948a 100644
--- a/lib/zdict.h
+++ b/lib/zdict.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -8,32 +8,43 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-#ifndef DICTBUILDER_H_001
-#define DICTBUILDER_H_001
-
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
+#ifndef ZSTD_ZDICT_H
+#define ZSTD_ZDICT_H
 
 /*======  Dependencies  ======*/
 #include   /* size_t */
 
 
 /* =====   ZDICTLIB_API : control library symbols visibility   ===== */
-#ifndef ZDICTLIB_VISIBILITY
-#  if defined(__GNUC__) && (__GNUC__ >= 4)
-#    define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#ifndef ZDICTLIB_VISIBLE
+   /* Backwards compatibility with old macro name */
+#  ifdef ZDICTLIB_VISIBILITY
+#    define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default")))
 #  else
-#    define ZDICTLIB_VISIBILITY
+#    define ZDICTLIB_VISIBLE
 #  endif
 #endif
+
+#ifndef ZDICTLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
+#    define ZDICTLIB_HIDDEN
+#  endif
+#endif
+
 #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
+#  define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE
 #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-#  define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#  define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
 #else
-#  define ZDICTLIB_API ZDICTLIB_VISIBILITY
+#  define ZDICTLIB_API ZDICTLIB_VISIBLE
 #endif
 
 /*******************************************************************************
@@ -201,9 +212,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
                                     const size_t* samplesSizes, unsigned nbSamples);
 
 typedef struct {
-    int      compressionLevel;   /*< optimize for a specific zstd compression level; 0 means default */
-    unsigned notificationLevel;  /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
-    unsigned dictID;             /*< force dictID value; 0 means auto mode (32-bits random value)
+    int      compressionLevel;   /**< optimize for a specific zstd compression level; 0 means default */
+    unsigned notificationLevel;  /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
+    unsigned dictID;             /**< force dictID value; 0 means auto mode (32-bits random value)
                                   *   NOTE: The zstd format reserves some dictionary IDs for future use.
                                   *         You may use them in private settings, but be warned that they
                                   *         may be used by zstd in a public dictionary registry in the future.
@@ -260,9 +271,21 @@ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictS
 ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
 ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
 
+#endif   /* ZSTD_ZDICT_H */
 
+#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC)
+#define ZSTD_ZDICT_H_STATIC
 
-#ifdef ZDICT_STATIC_LINKING_ONLY
+/* This can be overridden externally to hide static symbols. */
+#ifndef ZDICTLIB_STATIC_API
+#  if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#    define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE
+#  elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#    define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE
+#  else
+#    define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE
+#  endif
+#endif
 
 /* ====================================================================================
  * The definitions in this section are considered experimental.
@@ -318,7 +341,7 @@ typedef struct {
  *        In general, it's recommended to provide a few thousands samples, though this can vary a lot.
  *        It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
  */
-ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
           void *dictBuffer, size_t dictBufferCapacity,
     const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
           ZDICT_cover_params_t parameters);
@@ -340,7 +363,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
  *          See ZDICT_trainFromBuffer() for details on failure modes.
  * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
  */
-ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
+ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
           void* dictBuffer, size_t dictBufferCapacity,
     const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
           ZDICT_cover_params_t* parameters);
@@ -361,7 +384,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
  *        In general, it's recommended to provide a few thousands samples, though this can vary a lot.
  *        It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
  */
-ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
                     size_t dictBufferCapacity, const void *samplesBuffer,
                     const size_t *samplesSizes, unsigned nbSamples,
                     ZDICT_fastCover_params_t parameters);
@@ -384,7 +407,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
  *          See ZDICT_trainFromBuffer() for details on failure modes.
  * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
  */
-ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
+ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
                     size_t dictBufferCapacity, const void* samplesBuffer,
                     const size_t* samplesSizes, unsigned nbSamples,
                     ZDICT_fastCover_params_t* parameters);
@@ -409,7 +432,7 @@ typedef struct {
  *        It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
  *  Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
  */
-ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy(
     void* dictBuffer, size_t dictBufferCapacity,
     const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
     ZDICT_legacy_params_t parameters);
@@ -421,32 +444,31 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
    or _CRT_SECURE_NO_WARNINGS in Visual.
    Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
 #ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
-#  define ZDICT_DEPRECATED(message) ZDICTLIB_API   /* disable deprecation warnings */
+#  define ZDICT_DEPRECATED(message) /* disable deprecation warnings */
 #else
 #  define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
 #  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
-#    define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
+#    define ZDICT_DEPRECATED(message) [[deprecated(message)]]
 #  elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
+#    define ZDICT_DEPRECATED(message) __attribute__((deprecated(message)))
 #  elif (ZDICT_GCC_VERSION >= 301)
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
+#    define ZDICT_DEPRECATED(message) __attribute__((deprecated))
 #  elif defined(_MSC_VER)
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message))
+#    define ZDICT_DEPRECATED(message) __declspec(deprecated(message))
 #  else
 #    pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
-#    define ZDICT_DEPRECATED(message) ZDICTLIB_API
+#    define ZDICT_DEPRECATED(message)
 #  endif
 #endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
 
 ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
+ZDICTLIB_STATIC_API
 size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
                                   const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
 
 
-#endif   /* ZDICT_STATIC_LINKING_ONLY */
+#endif   /* ZSTD_ZDICT_H_STATIC */
 
 #if defined (__cplusplus)
 }
 #endif
-
-#endif   /* DICTBUILDER_H_001 */
diff --git a/lib/zstd.h b/lib/zstd.h
index 1867efc93..91a3679a1 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,21 +14,31 @@ extern "C" {
 #ifndef ZSTD_H_235446
 #define ZSTD_H_235446
 
-/* ======   Dependency   ======*/
+/* ======   Dependencies   ======*/
 #include    /* INT_MAX */
 #include    /* size_t */
 
 
 /* =====   ZSTDLIB_API : control library symbols visibility   ===== */
 #ifndef ZSTDLIB_VISIBLE
-#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+   /* Backwards compatibility with old macro name */
+#  ifdef ZSTDLIB_VISIBILITY
+#    define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
 #    define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
-#    define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
 #  else
 #    define ZSTDLIB_VISIBLE
+#  endif
+#endif
+
+#ifndef ZSTDLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
 #    define ZSTDLIB_HIDDEN
 #  endif
 #endif
+
 #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
 #  define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
 #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
@@ -96,7 +106,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    5
-#define ZSTD_VERSION_RELEASE  3
+#define ZSTD_VERSION_RELEASE  4
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 
 /*! ZSTD_versionNumber() :
@@ -201,8 +211,30 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
 
 
 /*======  Helper functions  ======*/
-#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-ZSTDLIB_API size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+/* ZSTD_compressBound() :
+ * maximum compressed size in worst case single-pass scenario.
+ * When invoking `ZSTD_compress()` or any other one-pass compression function,
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ * as it eliminates one potential failure scenario,
+ * aka not enough room in dst buffer to write the compressed frame.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
+ *        In which case, ZSTD_compressBound() will return an error code
+ *        which can be tested using ZSTD_isError().
+ *
+ * ZSTD_COMPRESSBOUND() :
+ * same as ZSTD_compressBound(), but as a macro.
+ * It can be used to produce constants, which can be useful for static allocation,
+ * for example to size a static array on stack.
+ * Will produce constant value 0 if srcSize too large.
+ */
+#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
+#define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
+ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+/* ZSTD_isError() :
+ * Most ZSTD_* functions returning a size_t value can be tested for error,
+ * using ZSTD_isError().
+ * @return 1 if error, 0 otherwise
+ */
 ZSTDLIB_API unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
 ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
 ZSTDLIB_API int         ZSTD_minCLevel(void);               /*!< minimum negative compression level allowed, requires v1.4.0+ */
@@ -446,6 +478,8 @@ typedef enum {
      * ZSTD_c_useBlockSplitter
      * ZSTD_c_useRowMatchFinder
      * ZSTD_c_prefetchCDictTables
+     * ZSTD_c_enableMatchFinderFallback
+     * ZSTD_c_maxBlockSize
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -465,7 +499,10 @@ typedef enum {
      ZSTD_c_experimentalParam13=1010,
      ZSTD_c_experimentalParam14=1011,
      ZSTD_c_experimentalParam15=1012,
-     ZSTD_c_experimentalParam16=1013
+     ZSTD_c_experimentalParam16=1013,
+     ZSTD_c_experimentalParam17=1014,
+     ZSTD_c_experimentalParam18=1015,
+     ZSTD_c_experimentalParam19=1016
 } ZSTD_cParameter;
 
 typedef struct {
@@ -528,7 +565,7 @@ typedef enum {
  *                  They will be used to compress next frame.
  *                  Resetting session never fails.
  *  - The parameters : changes all parameters back to "default".
- *                  This removes any reference to any dictionary too.
+ *                  This also removes any reference to any dictionary or external matchfinder.
  *                  Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
  *                  otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
  *  - Both : similar to resetting the session, followed by resetting parameters.
@@ -578,13 +615,15 @@ typedef enum {
      * ZSTD_d_stableOutBuffer
      * ZSTD_d_forceIgnoreChecksum
      * ZSTD_d_refMultipleDDicts
+     * ZSTD_d_disableHuffmanAssembly
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
      ZSTD_d_experimentalParam1=1000,
      ZSTD_d_experimentalParam2=1001,
      ZSTD_d_experimentalParam3=1002,
-     ZSTD_d_experimentalParam4=1003
+     ZSTD_d_experimentalParam4=1003,
+     ZSTD_d_experimentalParam5=1004
 
 } ZSTD_dParameter;
 
@@ -763,8 +802,6 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
  * This following is a legacy streaming API, available since v1.0+ .
  * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
  * It is redundant, but remains fully supported.
- * Streaming in combination with advanced parameters and dictionary compression
- * can only be used through the new API.
  ******************************************************************************/
 
 /*!
@@ -773,6 +810,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
  *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
  *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *
+ * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
+ * to compress with a dictionary.
  */
 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
 /*!
@@ -990,8 +1030,9 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
  *           meaning "return to no-dictionary mode".
- *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
- *           To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
+ *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
+ *           until parameters are reset, a new dictionary is loaded, or the dictionary
+ *           is explicitly invalidated by loading a NULL dictionary.
  *  Note 2 : Loading a dictionary involves building tables.
  *           It's also a CPU consuming operation, with non-negligible impact on latency.
  *           Tables are dependent on compression parameters, and for this reason,
@@ -1004,7 +1045,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
 
 /*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
- *  Reference a prepared dictionary, to be used for all next compressed frames.
+ *  Reference a prepared dictionary, to be used for all future compressed frames.
  *  Note that compression parameters are enforced from within CDict,
  *  and supersede any compression parameter previously set within CCtx.
  *  The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
@@ -1039,9 +1080,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
                                  const void* prefix, size_t prefixSize);
 
 /*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
- *  Create an internal DDict from dict buffer,
- *  to be used to decompress next frames.
- *  The dictionary remains valid for all future frames, until explicitly invalidated.
+ *  Create an internal DDict from dict buffer, to be used to decompress all future frames.
+ *  The dictionary remains valid for all future frames, until explicitly invalidated, or
+ *  a new dictionary is loaded.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
  *            meaning "return to no-dictionary mode".
@@ -1065,9 +1106,10 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
  *  The memory for the table is allocated on the first call to refDDict, and can be
  *  freed with ZSTD_freeDCtx().
  *
+ *  If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
+ *  will be managed, and referencing a dictionary effectively "discards" any previous one.
+ *
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Note 1 : Currently, only one dictionary can be managed.
- *           Referencing a new dictionary effectively "discards" any previous one.
  *  Special: referencing a NULL DDict means "return to no-dictionary mode".
  *  Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
  */
@@ -1164,6 +1206,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 #define ZSTD_TARGETLENGTH_MIN     0   /* note : comparing this constant to an unsigned results in a tautological test */
 #define ZSTD_STRATEGY_MIN        ZSTD_fast
 #define ZSTD_STRATEGY_MAX        ZSTD_btultra2
+#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
 
 
 #define ZSTD_OVERLAPLOG_MIN       0
@@ -1391,6 +1434,51 @@ ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size
  *           or an error code (if srcSize is too small) */
 ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
 
+/*! ZSTD_decompressionMargin() :
+ * Zstd supports in-place decompression, where the input and output buffers overlap.
+ * In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
+ * and the input buffer must be at the end of the output buffer.
+ *
+ *  _______________________ Output Buffer ________________________
+ * |                                                              |
+ * |                                        ____ Input Buffer ____|
+ * |                                       |                      |
+ * v                                       v                      v
+ * |---------------------------------------|-----------|----------|
+ * ^                                                   ^          ^
+ * |___________________ Output_Size ___________________|_ Margin _|
+ *
+ * NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
+ * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
+ * ZSTD_decompressDCtx().
+ * NOTE: This function supports multi-frame input.
+ *
+ * @param src The compressed frame(s)
+ * @param srcSize The size of the compressed frame(s)
+ * @returns The decompression margin or an error that can be checked with ZSTD_isError().
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
+
+/*! ZSTD_DECOMPRESS_MARGIN() :
+ * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
+ * the compressed frame, compute it from the original size and the blockSizeLog.
+ * See ZSTD_decompressionMargin() for details.
+ *
+ * WARNING: This macro does not support multi-frame input, the input must be a single
+ * zstd frame. If you need that support use the function, or implement it yourself.
+ *
+ * @param originalSize The original uncompressed size of the data.
+ * @param blockSize    The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
+ *                     Unless you explicitly set the windowLog smaller than
+ *                     ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
+ */
+#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)(                                              \
+        ZSTD_FRAMEHEADERSIZE_MAX                                                              /* Frame header */ + \
+        4                                                                                         /* checksum */ + \
+        ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
+        (blockSize)                                                                    /* One block of margin */   \
+    ))
+
 typedef enum {
   ZSTD_sf_noBlockDelimiters = 0,         /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
   ZSTD_sf_explicitBlockDelimiters = 1    /* Representation of ZSTD_Sequence contains explicit block delimiters */
@@ -1536,8 +1624,11 @@ ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
  *  and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
  *  Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
  *
- *  Note 2 : only single-threaded compression is supported.
+ *  Note : only single-threaded compression is supported.
  *  ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *
+ *  Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the external matchfinder API at this time.
+ *  Size estimates assume that no external matchfinder is registered.
  */
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@@ -1556,7 +1647,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
  *  or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
  *  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
  *         an internal ?Dict will be created, which additional size is not estimated here.
- *         In this case, get total size by adding ZSTD_estimate?DictSize */
+ *         In this case, get total size by adding ZSTD_estimate?DictSize
+ *  Note 2 : only single-threaded compression is supported.
+ *  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *  Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the external matchfinder API at this time.
+ *  Size estimates assume that no external matchfinder is registered.
+ */
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
@@ -1706,6 +1802,13 @@ ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
  *  This function never fails (wide contract) */
 ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
 
+/*! ZSTD_CCtx_setCParams() :
+ *  Set all parameters provided within @cparams into the working @cctx.
+ *  Note : if modifying parameters during compression (MT mode only),
+ *         note that changes to the .windowLog parameter will be ignored.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
+
 /*! ZSTD_compress_advanced() :
  *  Note : this function is now DEPRECATED.
  *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
@@ -1713,10 +1816,10 @@ ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressio
 ZSTD_DEPRECATED("use ZSTD_compress2")
 ZSTDLIB_STATIC_API
 size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
-                                          void* dst, size_t dstCapacity,
-                                    const void* src, size_t srcSize,
-                                    const void* dict,size_t dictSize,
-                                          ZSTD_parameters params);
+                              void* dst, size_t dstCapacity,
+                        const void* src, size_t srcSize,
+                        const void* dict,size_t dictSize,
+                              ZSTD_parameters params);
 
 /*! ZSTD_compress_usingCDict_advanced() :
  *  Note : this function is now DEPRECATED.
@@ -2010,6 +2113,55 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
  */
 #define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
 
+/* ZSTD_c_enableMatchFinderFallback
+ * Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
+ *
+ * Controls whether zstd will fall back to an internal matchfinder if an
+ * external matchfinder is registered and returns an error code. This fallback is
+ * block-by-block: the internal matchfinder will only be called for blocks where
+ * the external matchfinder returns an error code. Fallback compression will
+ * follow any other cParam settings, such as compression level, the same as in a
+ * normal (fully-internal) compression operation.
+ *
+ * The user is strongly encouraged to read the full external matchfinder API
+ * documentation (below) before setting this parameter. */
+#define ZSTD_c_enableMatchFinderFallback ZSTD_c_experimentalParam17
+
+/* ZSTD_c_maxBlockSize
+ * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+ *
+ * This parameter can be used to set an upper bound on the blocksize
+ * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
+ * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
+ * compressBound() innacurate). Only currently meant to be used for testing.
+ *
+ */
+#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
+
+/* ZSTD_c_searchForExternalRepcodes
+ * This parameter affects how zstd parses external sequences, such as sequences
+ * provided through the compressSequences() API or from an external matchfinder.
+ *
+ * If set to ZSTD_ps_enable, the library will check for repeated offsets in
+ * external sequences, even if those repcodes are not explicitly indicated in
+ * the "rep" field. Note that this is the only way to exploit repcode matches
+ * while using compressSequences() or an external matchfinder, since zstd
+ * currently ignores the "rep" field of external sequences.
+ *
+ * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
+ * external sequences, regardless of whether the "rep" field has been set. This
+ * reduces sequence compression overhead by about 25% while sacrificing some
+ * compression ratio.
+ *
+ * The default value is ZSTD_ps_auto, for which the library will enable/disable
+ * based on compression level.
+ *
+ * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
+ * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
+ */
+#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
+
 /*! ZSTD_CCtx_getParameter() :
  *  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
  *  and store it into int* value.
@@ -2219,6 +2371,17 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
  */
 #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
 
+/* ZSTD_d_disableHuffmanAssembly
+ * Set to 1 to disable the Huffman assembly implementation.
+ * The default value is 0, which allows zstd to use the Huffman assembly
+ * implementation if available.
+ *
+ * This parameter can be used to disable Huffman assembly at runtime.
+ * If you want to disable it at compile time you can define the macro
+ * ZSTD_DISABLE_ASM.
+ */
+#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5
+
 
 /*! ZSTD_DCtx_setFormat() :
  *  This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
@@ -2408,8 +2571,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
  *     ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
  *
  * note: no dictionary will be used if dict == NULL or dictSize < 8
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
 ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 
 /*!
@@ -2419,8 +2582,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const vo
  *     ZSTD_DCtx_refDDict(zds, ddict);
  *
  * note : ddict is referenced, it must outlive decompression session
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
 ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
 
 /*!
@@ -2429,8 +2592,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const Z
  *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
  *
  * re-use decompression parameters from previous init; saves dictionary loading
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
 ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 
 
@@ -2476,8 +2639,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLev
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
 
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 
 ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@@ -2501,8 +2664,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
   Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
   Data fragment must be large enough to ensure successful decoding.
  `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
-  @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
-           >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
+  result  : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
+           >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
            errorCode, which can be tested using ZSTD_isError().
 
   It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
@@ -2521,7 +2684,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
 
   The most memory efficient way is to use a round buffer of sufficient size.
   Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
-  which can @return an error code if required value is too large for current system (in 32-bits mode).
+  which can return an error code if required value is too large for current system (in 32-bits mode).
   In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
   up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
   which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
@@ -2541,7 +2704,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
   ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
   ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
 
- @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+  result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
   It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
   It can also be an error code, which can be tested with ZSTD_isError().
 
@@ -2573,6 +2736,8 @@ typedef struct {
     unsigned headerSize;
     unsigned dictID;
     unsigned checksumFlag;
+    unsigned _reserved1;
+    unsigned _reserved2;
 } ZSTD_frameHeader;
 
 /*! ZSTD_getFrameHeader() :
@@ -2640,6 +2805,157 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_
 ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
 
 
+/* ********************** EXTERNAL MATCHFINDER API **********************
+ *
+ * *** OVERVIEW ***
+ * This API allows users to replace the zstd internal block-level matchfinder
+ * with an external matchfinder function. Potential applications of the API
+ * include hardware-accelerated matchfinders and matchfinders specialized to
+ * particular types of data.
+ *
+ * See contrib/externalMatchfinder for an example program employing the
+ * external matchfinder API.
+ *
+ * *** USAGE ***
+ * The user is responsible for implementing a function of type
+ * ZSTD_externalMatchFinder_F. For each block, zstd will pass the following
+ * arguments to the user-provided function:
+ *
+ *   - externalMatchState: a pointer to a user-managed state for the external
+ *     matchfinder.
+ *
+ *   - outSeqs, outSeqsCapacity: an output buffer for sequences produced by the
+ *     external matchfinder. outSeqsCapacity is guaranteed >=
+ *     ZSTD_sequenceBound(srcSize). The memory backing outSeqs is managed by
+ *     the CCtx.
+ *
+ *   - src, srcSize: an input buffer which the external matchfinder must parse
+ *     into sequences. srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
+ *
+ *   - dict, dictSize: a history buffer, which may be empty, which the external
+ *     matchfinder may reference as it produces sequences for the src buffer.
+ *     Currently, zstd will always pass dictSize == 0 into external matchfinders,
+ *     but this will change in the future.
+ *
+ *   - compressionLevel: a signed integer representing the zstd compression level
+ *     set by the user for the current operation. The external matchfinder may
+ *     choose to use this information to change its compression strategy and
+ *     speed/ratio tradeoff. Note: The compression level does not reflect zstd
+ *     parameters set through the advanced API.
+ *
+ *   - windowSize: a size_t representing the maximum allowed offset for external
+ *     sequences. Note that sequence offsets are sometimes allowed to exceed the
+ *     windowSize if a dictionary is present, see doc/zstd_compression_format.md
+ *     for details.
+ *
+ * The user-provided function shall return a size_t representing the number of
+ * sequences written to outSeqs. This return value will be treated as an error
+ * code if it is greater than outSeqsCapacity. The return value must be non-zero
+ * if srcSize is non-zero. The ZSTD_EXTERNAL_MATCHFINDER_ERROR macro is provided
+ * for convenience, but any value greater than outSeqsCapacity will be treated as
+ * an error code.
+ *
+ * If the user-provided function does not return an error code, the sequences
+ * written to outSeqs must be a valid parse of the src buffer. Data corruption may
+ * occur if the parse is not valid. A parse is defined to be valid if the
+ * following conditions hold:
+ *   - The sum of matchLengths and literalLengths is equal to srcSize.
+ *   - All sequences in the parse have matchLength != 0, except for the final
+ *     sequence. matchLength is not constrained for the final sequence.
+ *   - All offsets respect the windowSize parameter as specified in
+ *     doc/zstd_compression_format.md.
+ *
+ * zstd will only validate these conditions (and fail compression if they do not
+ * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
+ * validation has a performance cost.
+ *
+ * If the user-provided function returns an error, zstd will either fall back
+ * to an internal matchfinder or fail the compression operation. The user can
+ * choose between the two behaviors by setting the
+ * ZSTD_c_enableMatchFinderFallback cParam. Fallback compression will follow any
+ * other cParam settings, such as compression level, the same as in a normal
+ * compression operation.
+ *
+ * The user shall instruct zstd to use a particular ZSTD_externalMatchFinder_F
+ * function by calling ZSTD_registerExternalMatchFinder(cctx, externalMatchState,
+ * externalMatchFinder). This setting will persist until the next parameter reset
+ * of the CCtx.
+ *
+ * The externalMatchState must be initialized by the user before calling
+ * ZSTD_registerExternalMatchFinder. The user is responsible for destroying the
+ * externalMatchState.
+ *
+ * *** LIMITATIONS ***
+ * External matchfinders are compatible with all zstd compression APIs which respect
+ * advanced parameters. However, there are three limitations:
+ *
+ * First, the ZSTD_c_enableLongDistanceMatching cParam is not supported.
+ * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with an
+ * external matchfinder.
+ *   - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in
+ *     some cases (see its documentation for details). Users must explicitly set
+ *     ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an
+ *     external matchfinder is registered.
+ *   - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
+ *     whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
+ *     check the docs on ZSTD_c_enableLongDistanceMatching whenever the external
+ *     matchfinder API is used in conjunction with advanced settings (like windowLog).
+ *
+ * Second, history buffers are not supported. Concretely, zstd will always pass
+ * dictSize == 0 to the external matchfinder (for now). This has two implications:
+ *   - Dictionaries are not supported. Compression will *not* fail if the user
+ *     references a dictionary, but the dictionary won't have any effect.
+ *   - Stream history is not supported. All compression APIs, including streaming
+ *     APIs, work with the external matchfinder, but the external matchfinder won't
+ *     receive any history from the previous block. Each block is an independent chunk.
+ *
+ * Third, multi-threading within a single compression is not supported. In other words,
+ * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external matchfinder is registered.
+ * Multi-threading across compressions is fine: simply create one CCtx per thread.
+ *
+ * Long-term, we plan to overcome all three limitations. There is no technical blocker to
+ * overcoming them. It is purely a question of engineering effort.
+ */
+
+#define ZSTD_EXTERNAL_MATCHFINDER_ERROR ((size_t)(-1))
+
+typedef size_t ZSTD_externalMatchFinder_F (
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+/*! ZSTD_registerExternalMatchFinder() :
+ * Instruct zstd to use an external matchfinder function.
+ *
+ * The externalMatchState must be initialized by the caller, and the caller is
+ * responsible for managing its lifetime. This parameter is sticky across
+ * compressions. It will remain set until the user explicitly resets compression
+ * parameters.
+ *
+ * External matchfinder registration is considered to be an "advanced parameter",
+ * part of the "advanced API". This means it will only have an effect on
+ * compression APIs which respect advanced parameters, such as compress2() and
+ * compressStream(). Older compression APIs such as compressCCtx(), which predate
+ * the introduction of "advanced parameters", will ignore any external matchfinder
+ * setting.
+ *
+ * The external matchfinder can be "cleared" by registering a NULL external
+ * matchfinder function pointer. This removes all limitations described above in
+ * the "LIMITATIONS" section of the API docs.
+ *
+ * The user is strongly encouraged to read the full API documentation (above)
+ * before calling this function. */
+ZSTDLIB_STATIC_API void
+ZSTD_registerExternalMatchFinder(
+  ZSTD_CCtx* cctx,
+  void* externalMatchState,
+  ZSTD_externalMatchFinder_F* externalMatchFinder
+);
+
 #endif   /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
 
 #if defined (__cplusplus)
diff --git a/lib/zstd_errors.h b/lib/zstd_errors.h
index 2ec0b0ab1..6a66bedcb 100644
--- a/lib/zstd_errors.h
+++ b/lib/zstd_errors.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -20,19 +20,31 @@ extern "C" {
 
 
 /* =====   ZSTDERRORLIB_API : control library symbols visibility   ===== */
-#ifndef ZSTDERRORLIB_VISIBILITY
-#  if defined(__GNUC__) && (__GNUC__ >= 4)
-#    define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#ifndef ZSTDERRORLIB_VISIBLE
+   /* Backwards compatibility with old macro name */
+#  ifdef ZSTDERRORLIB_VISIBILITY
+#    define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY
+#  elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default")))
 #  else
-#    define ZSTDERRORLIB_VISIBILITY
+#    define ZSTDERRORLIB_VISIBLE
 #  endif
 #endif
+
+#ifndef ZSTDERRORLIB_HIDDEN
+#  if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
+#    define ZSTDERRORLIB_HIDDEN
+#  endif
+#endif
+
 #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
+#  define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE
 #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-#  define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#  define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
 #else
-#  define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
+#  define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
 #endif
 
 /*-*********************************************
@@ -58,10 +70,12 @@ typedef enum {
   ZSTD_error_frameParameter_windowTooLarge = 16,
   ZSTD_error_corruption_detected = 20,
   ZSTD_error_checksum_wrong      = 22,
+  ZSTD_error_literals_headerWrong = 24,
   ZSTD_error_dictionary_corrupted      = 30,
   ZSTD_error_dictionary_wrong          = 32,
   ZSTD_error_dictionaryCreation_failed = 34,
   ZSTD_error_parameter_unsupported   = 40,
+  ZSTD_error_parameter_combination_unsupported = 41,
   ZSTD_error_parameter_outOfBound    = 42,
   ZSTD_error_tableLog_tooLarge       = 44,
   ZSTD_error_maxSymbolValue_tooLarge = 46,
@@ -74,11 +88,15 @@ typedef enum {
   ZSTD_error_dstSize_tooSmall = 70,
   ZSTD_error_srcSize_wrong    = 72,
   ZSTD_error_dstBuffer_null   = 74,
+  ZSTD_error_noForwardProgress_destFull = 80,
+  ZSTD_error_noForwardProgress_inputEmpty = 82,
   /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
   ZSTD_error_frameIndex_tooLarge = 100,
   ZSTD_error_seekableIO          = 102,
   ZSTD_error_dstBuffer_wrong     = 104,
   ZSTD_error_srcBuffer_wrong     = 105,
+  ZSTD_error_externalMatchFinder_failed = 106,
+  ZSTD_error_externalSequences_invalid = 107,
   ZSTD_error_maxCode = 120  /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
 } ZSTD_ErrorCode;
 
diff --git a/programs/Makefile b/programs/Makefile
index da9883a2e..fcff41dc1 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -231,9 +231,12 @@ zstd-dll : zstd
 
 ## zstd-pgo: zstd executable optimized with PGO.
 .PHONY: zstd-pgo
+zstd-pgo : LLVM_PROFDATA?=llvm-profdata
+zstd-pgo : PROF_GENERATE_FLAGS=-fprofile-generate $(if $(findstring gcc,$(CC)),-fprofile-dir=.)
+zstd-pgo : PROF_USE_FLAGS=-fprofile-use $(if $(findstring gcc,$(CC)),-fprofile-dir=. -Werror=missing-profile -Wno-error=coverage-mismatch)
 zstd-pgo :
 	$(MAKE) clean HASH_DIR=$(HASH_DIR)
-	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-generate
+	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_GENERATE_FLAGS)"
 	./zstd -b19i1 $(PROFILE_WITH)
 	./zstd -b16i1 $(PROFILE_WITH)
 	./zstd -b9i2 $(PROFILE_WITH)
@@ -245,8 +248,8 @@ ifndef BUILD_DIR
 else
 	$(RM) zstd $(BUILD_DIR)/zstd $(BUILD_DIR)/*.o
 endif
-	case $(CC) in *clang*) if ! [ -e default.profdata ]; then llvm-profdata merge -output=default.profdata default*.profraw; fi ;; esac
-	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-use
+	case $(CC) in *clang*) if ! [ -e default.profdata ]; then $(LLVM_PROFDATA) merge -output=default.profdata default*.profraw; fi ;; esac
+	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_USE_FLAGS)"
 
 ## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
 CLEAN += zstd-small zstd-frugal
@@ -276,7 +279,7 @@ generate_res: $(RES64_FILE) $(RES32_FILE)
 
 ifneq (,$(filter Windows%,$(OS)))
 RC ?= windres
-# http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
+# https://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
 $(RES64_FILE): windres/zstd.rc
 	$(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-x86-64
 $(RES32_FILE): windres/zstd.rc
diff --git a/programs/README.md b/programs/README.md
index c192b8026..1b9f47cbb 100644
--- a/programs/README.md
+++ b/programs/README.md
@@ -276,7 +276,7 @@ compression speed (for lower levels) with minimal change in compression ratio.
 
 The below table illustrates this on the [Silesia compression corpus].
 
-[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
+[Silesia compression corpus]: https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia
 
 | Method | Compression ratio | Compression speed | Decompression speed  |
 |:-------|------------------:|------------------:|---------------------:|
diff --git a/programs/benchfn.c b/programs/benchfn.c
index 1aadbdd91..8e6726f8d 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -229,9 +229,9 @@ BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
             cont->timeSpent_ns += (unsigned long long)loopDuration_ns;
 
             /* estimate nbLoops for next run to last approximately 1 second */
-            if (loopDuration_ns > (runBudget_ns / 50)) {
+            if (loopDuration_ns > ((double)runBudget_ns / 50)) {
                 double const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
-                cont->nbLoops = (unsigned)(runBudget_ns / fastestRun_ns) + 1;
+                cont->nbLoops = (unsigned)((double)runBudget_ns / fastestRun_ns) + 1;
             } else {
                 /* previous run was too short : blindly increase workload by x multiplier */
                 const unsigned multiplier = 10;
@@ -239,7 +239,7 @@ BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
                 cont->nbLoops *= multiplier;
             }
 
-            if(loopDuration_ns < runTimeMin_ns) {
+            if(loopDuration_ns < (double)runTimeMin_ns) {
                 /* don't report results for which benchmark run time was too small : increased risks of rounding errors */
                 assert(completed == 0);
                 continue;
diff --git a/programs/benchfn.h b/programs/benchfn.h
index 99d13ac47..1bd93d135 100644
--- a/programs/benchfn.h
+++ b/programs/benchfn.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 6ceca020c..a76db5f37 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -327,26 +327,31 @@ BMK_benchMemAdvancedNoAlloc(
     /* init */
     memset(&benchResult, 0, sizeof(benchResult));
     if (strlen(displayName)>17) displayName += strlen(displayName) - 17;   /* display last 17 characters */
-    if (adv->mode == BMK_decodeOnly) {  /* benchmark only decompression : source must be already compressed */
+    if (adv->mode == BMK_decodeOnly) {
+        /* benchmark only decompression : source must be already compressed */
         const char* srcPtr = (const char*)srcBuffer;
         U64 totalDSize64 = 0;
         U32 fileNb;
         for (fileNb=0; fileNb decodedSize) {  /* size_t overflow */
+                RETURN_ERROR(32, BMK_benchOutcome_t, "decompressed size is too large for local system");
+            }
             *resultBufferPtr = malloc(decodedSize);
             if (!(*resultBufferPtr)) {
-                RETURN_ERROR(33, BMK_benchOutcome_t, "not enough memory");
-            }
-            if (totalDSize64 > decodedSize) {  /* size_t overflow */
-                free(*resultBufferPtr);
-                RETURN_ERROR(32, BMK_benchOutcome_t, "original size is too large");
+                RETURN_ERROR(33, BMK_benchOutcome_t, "allocation error: not enough memory");
             }
             cSize = srcSize;
             srcSize = decodedSize;
@@ -387,12 +392,9 @@ BMK_benchMemAdvancedNoAlloc(
         RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
     }
 
-#if defined(UTIL_TIME_USES_C90_CLOCK)
-    if (adv->nbWorkers > 1) {
-        OUTPUTLEVEL(2, "Warning : time measurements restricted to C90 clock_t. \n")
-        OUTPUTLEVEL(2, "Warning : using C90 clock_t leads to incorrect measurements in multithreading mode. \n")
+    if (!UTIL_support_MT_measurements() && adv->nbWorkers > 1) {
+        OUTPUTLEVEL(2, "Warning : time measurements may be incorrect in multithreading mode... \n")
     }
-#endif
 
     /* Bench */
     {   U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
@@ -449,7 +451,7 @@ BMK_benchMemAdvancedNoAlloc(
                 BMK_runOutcome_t const cOutcome = BMK_benchTimedFn( timeStateCompress, cbp);
 
                 if (!BMK_isSuccessful_runOutcome(cOutcome)) {
-                    return BMK_benchOutcome_error();
+                    RETURN_ERROR(30, BMK_benchOutcome_t, "compression error");
                 }
 
                 {   BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome);
@@ -477,7 +479,7 @@ BMK_benchMemAdvancedNoAlloc(
                 BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp);
 
                 if(!BMK_isSuccessful_runOutcome(dOutcome)) {
-                    return BMK_benchOutcome_error();
+                    RETURN_ERROR(30, BMK_benchOutcome_t, "decompression error");
                 }
 
                 {   BMK_runTime_t const dResult = BMK_extract_runTime(dOutcome);
@@ -601,7 +603,7 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
 
     void* resultBuffer = srcSize ? malloc(srcSize) : NULL;
 
-    int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
+    int const allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
         !cSizes || !cCapacities || !resPtrs || !resSizes ||
         !timeStateCompress || !timeStateDecompress ||
         !cctx || !dctx ||
diff --git a/programs/benchzstd.h b/programs/benchzstd.h
index 11ac85da7..aa683dfc2 100644
--- a/programs/benchzstd.h
+++ b/programs/benchzstd.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/datagen.c b/programs/datagen.c
index 3b4f9e5c7..ddc690bb1 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/datagen.h b/programs/datagen.h
index b76ae2a22..ca7270006 100644
--- a/programs/datagen.h
+++ b/programs/datagen.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/dibio.c b/programs/dibio.c
index 8643bc378..26ebe5ca1 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -274,21 +274,20 @@ static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t
     int n;
     memset(&fs, 0, sizeof(fs));
 
-    // We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX
+    /* We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX */
     assert( chunkSize <= SAMPLESIZE_MAX );
 
     for (n=0; n 0)
-      {
-        // TODO: is there a minimum sample size? Can we have a 1-byte sample?
+      if (chunkSize > 0) {
+        /* TODO: is there a minimum sample size? Can we have a 1-byte sample? */
         fs.nbSamples += (int)((fileSize + chunkSize-1) / chunkSize);
         fs.totalSizeToLoad += fileSize;
       }
diff --git a/programs/dibio.h b/programs/dibio.h
index 666c1e661..a96104c36 100644
--- a/programs/dibio.h
+++ b/programs/dibio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/fileio.c b/programs/fileio.c
index e80d37011..0da18484b 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -28,6 +28,7 @@
 #include       /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */
 #include      /* malloc, free */
 #include      /* strcmp, strlen */
+#include        /* clock_t, to measure process time */
 #include       /* O_WRONLY */
 #include 
 #include       /* errno */
@@ -40,6 +41,10 @@
 #  include 
 #endif
 
+#if (PLATFORM_POSIX_VERSION > 0)
+#  include 
+#endif
+
 #include "fileio.h"
 #include "fileio_asyncio.h"
 #include "fileio_common.h"
@@ -113,11 +118,15 @@ char const* FIO_lzmaVersion(void)
 #define FNSPACE 30
 
 /* Default file permissions 0666 (modulated by umask) */
+/* Temporary restricted file permissions are used when we're going to
+ * chmod/chown at the end of the operation. */
 #if !defined(_WIN32)
 /* These macros aren't defined on windows. */
 #define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)
+#define TEMPORARY_FILE_PERMISSIONS (S_IRUSR|S_IWUSR)
 #else
 #define DEFAULT_FILE_PERMISSIONS (0666)
+#define TEMPORARY_FILE_PERMISSIONS (0600)
 #endif
 
 /*-************************************
@@ -249,6 +258,18 @@ struct FIO_ctx_s {
     size_t totalBytesOutput;
 };
 
+static int FIO_shouldDisplayFileSummary(FIO_ctx_t const* fCtx)
+{
+    return fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3;
+}
+
+static int FIO_shouldDisplayMultipleFileSummary(FIO_ctx_t const* fCtx)
+{
+    int const shouldDisplay = (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1);
+    assert(shouldDisplay || FIO_shouldDisplayFileSummary(fCtx) || fCtx->nbFilesProcessed == 0);
+    return shouldDisplay;
+}
+
 
 /*-*************************************
 *  Parameters: Initialization
@@ -345,7 +366,7 @@ void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictID
 
 void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; }
 
-void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); }
+void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, int flag) { prefs->removeSrcFile = (flag!=0); }
 
 void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; }
 
@@ -518,26 +539,26 @@ static int FIO_removeFile(const char* path)
 /** FIO_openSrcFile() :
  *  condition : `srcFileName` must be non-NULL. `prefs` may be NULL.
  * @result : FILE* to `srcFileName`, or NULL if it fails */
-static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName)
+static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName, stat_t* statbuf)
 {
-    stat_t statbuf;
     int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0;
     assert(srcFileName != NULL);
+    assert(statbuf != NULL);
     if (!strcmp (srcFileName, stdinmark)) {
         DISPLAYLEVEL(4,"Using stdin for input \n");
         SET_BINARY_MODE(stdin);
         return stdin;
     }
 
-    if (!UTIL_stat(srcFileName, &statbuf)) {
+    if (!UTIL_stat(srcFileName, statbuf)) {
         DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n",
                         srcFileName, strerror(errno));
         return NULL;
     }
 
-    if (!UTIL_isRegularFileStat(&statbuf)
-     && !UTIL_isFIFOStat(&statbuf)
-     && !(allowBlockDevices && UTIL_isBlockDevStat(&statbuf))
+    if (!UTIL_isRegularFileStat(statbuf)
+     && !UTIL_isFIFOStat(statbuf)
+     && !(allowBlockDevices && UTIL_isBlockDevStat(statbuf))
     ) {
         DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
                         srcFileName);
@@ -595,7 +616,7 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
         if (!prefs->overwrite) {
             if (g_display_prefs.displayLevel <= 1) {
                 /* No interaction possible */
-                DISPLAY("zstd: %s already exists; not overwritten  \n",
+                DISPLAYLEVEL(1, "zstd: %s already exists; not overwritten  \n",
                         dstFileName);
                 return NULL;
             }
@@ -649,23 +670,23 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
  * @return : loaded size
  *  if fileName==NULL, returns 0 and a NULL pointer
  */
-static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs)
+static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
 {
     FILE* fileHandle;
     U64 fileSize;
-    stat_t statbuf;
 
     assert(bufferPtr != NULL);
+    assert(dictFileStat != NULL);
     *bufferPtr = NULL;
     if (fileName == NULL) return 0;
 
     DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
 
-    if (!UTIL_stat(fileName, &statbuf)) {
+    if (!UTIL_stat(fileName, dictFileStat)) {
         EXM_THROW(31, "Stat failed on dictionary file %s: %s", fileName, strerror(errno));
     }
 
-    if (!UTIL_isRegularFileStat(&statbuf)) {
+    if (!UTIL_isRegularFileStat(dictFileStat)) {
         EXM_THROW(32, "Dictionary %s must be a regular file.", fileName);
     }
 
@@ -675,7 +696,7 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p
         EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
     }
 
-    fileSize = UTIL_getFileSizeStat(&statbuf);
+    fileSize = UTIL_getFileSizeStat(dictFileStat);
     {
         size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
         if (fileSize >  dictSizeMax) {
@@ -695,6 +716,54 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p
     return (size_t)fileSize;
 }
 
+/*! FIO_createDictBuffer() :
+ *  creates a buffer, pointed by `*bufferPtr` using mmap,
+ *  loads entire `filename` content into it.
+ * @return : loaded size
+ *  if fileName==NULL, returns 0 and a NULL pointer
+ */
+static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
+{
+    int fileHandle;
+    U64 fileSize;
+
+    assert(bufferPtr != NULL);
+    assert(dictFileStat != NULL);
+    *bufferPtr = NULL;
+    if (fileName == NULL) return 0;
+
+    DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
+
+     if (!UTIL_stat(fileName, dictFileStat)) {
+        EXM_THROW(31, "Stat failed on dictionary file %s: %s", fileName, strerror(errno));
+    }
+
+    if (!UTIL_isRegularFileStat(dictFileStat)) {
+        EXM_THROW(32, "Dictionary %s must be a regular file.", fileName);
+    }
+
+    fileHandle = open(fileName, O_RDONLY);
+
+    if (fileHandle == -1) {
+        EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
+    }
+
+    fileSize = UTIL_getFileSizeStat(dictFileStat);
+
+    {
+        size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
+        if (fileSize >  dictSizeMax) {
+            EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
+                            fileName,  (unsigned)dictSizeMax);   /* avoid extreme cases */
+        }
+    }
+
+    *bufferPtr = mmap(NULL, (size_t)fileSize, PROT_READ, MAP_PRIVATE, fileHandle, 0);
+
+    close(fileHandle);
+    return (size_t)fileSize;
+}
+
 
 
 /* FIO_checkFilenameCollisions() :
@@ -706,7 +775,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
 
     filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles);
     if (!filenameTableSorted) {
-        DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
+        DISPLAYLEVEL(1, "Allocation error during filename collision checking \n");
         return 1;
     }
 
@@ -723,7 +792,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
     prevElem = filenameTableSorted[0];
     for (u = 1; u < nbFiles; ++u) {
         if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
-            DISPLAY("WARNING: Two files have same filename: %s\n", prevElem);
+            DISPLAYLEVEL(2, "WARNING: Two files have same filename: %s\n", prevElem);
         }
         prevElem = filenameTableSorted[u];
     }
@@ -806,45 +875,89 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
     FIO_setMemLimit(prefs, (unsigned)maxSize);
 }
 
-/* FIO_removeMultiFilesWarning() :
+/* FIO_multiFilesConcatWarning() :
+ * This function handles logic when processing multiple files with -o or -c, displaying the appropriate warnings/prompts.
  * Returns 1 if the console should abort, 0 if console should proceed.
- * This function handles logic when processing multiple files with -o, displaying the appropriate warnings/prompts.
  *
- * If -f is specified, or there is just 1 file, zstd will always proceed as usual.
- * If --rm is specified, there will be a prompt asking for user confirmation.
- *         If -f is specified with --rm, zstd will proceed as usual
- *         If -q is specified with --rm, zstd will abort pre-emptively
- *         If neither flag is specified, zstd will prompt the user for confirmation to proceed.
- * If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q).
- * However, if the output is stdout, we will always abort rather than displaying the warning prompt.
+ * If output is stdout or test mode is active, check that `--rm` disabled.
+ *
+ * If there is just 1 file to process, zstd will proceed as usual.
+ * If each file get processed into its own separate destination file, proceed as usual.
+ *
+ * When multiple files are processed into a single output,
+ * display a warning message, then disable --rm if it's set.
+ *
+ * If -f is specified or if output is stdout, just proceed.
+ * If output is set with -o, prompt for confirmation.
  */
-static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* const prefs, const char* outFileName, int displayLevelCutoff)
+static int FIO_multiFilesConcatWarning(const FIO_ctx_t* fCtx, FIO_prefs_t* prefs, const char* outFileName, int displayLevelCutoff)
 {
-    int error = 0;
-    if (fCtx->nbFilesTotal > 1 && !prefs->overwrite) {
-        if (g_display_prefs.displayLevel <= displayLevelCutoff) {
-            if (prefs->removeSrcFile) {
-                DISPLAYLEVEL(1, "zstd: Aborting... not deleting files and processing into dst: %s\n", outFileName);
-                error =  1;
-            }
-        } else {
-            if (!strcmp(outFileName, stdoutmark)) {
-                DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. \n");
-            } else {
-                DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s \n", outFileName);
-            }
-            DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate the original directory tree. \n")
-            if (prefs->removeSrcFile) {
-                if (fCtx->hasStdoutOutput) {
-                    DISPLAYLEVEL(1, "Aborting. Use -f if you really want to delete the files and output to stdout\n");
-                    error = 1;
-                } else {
-                    error = g_display_prefs.displayLevel > displayLevelCutoff && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput);
-                }
-            }
-        }
+    if (fCtx->hasStdoutOutput) {
+        if (prefs->removeSrcFile)
+            /* this should not happen ; hard fail, to protect user's data
+             * note: this should rather be an assert(), but we want to be certain that user's data will not be wiped out in case it nonetheless happen */
+            EXM_THROW(43, "It's not allowed to remove input files when processed output is piped to stdout. "
+                "This scenario is not supposed to be possible. "
+                "This is a programming error. File an issue for it to be fixed.");
     }
-    return error;
+    if (prefs->testMode) {
+        if (prefs->removeSrcFile)
+            /* this should not happen ; hard fail, to protect user's data
+             * note: this should rather be an assert(), but we want to be certain that user's data will not be wiped out in case it nonetheless happen */
+            EXM_THROW(43, "Test mode shall not remove input files! "
+                 "This scenario is not supposed to be possible. "
+                 "This is a programming error. File an issue for it to be fixed.");
+        return 0;
+    }
+
+    if (fCtx->nbFilesTotal == 1) return 0;
+    assert(fCtx->nbFilesTotal > 1);
+
+    if (!outFileName) return 0;
+
+    if (fCtx->hasStdoutOutput) {
+        DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. \n");
+    } else {
+        DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s \n", outFileName);
+    }
+    DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate original file names nor directory structure. \n")
+
+    /* multi-input into single output : --rm is not allowed */
+    if (prefs->removeSrcFile) {
+        DISPLAYLEVEL(2, "Since it's a destructive operation, input files will not be removed. \n");
+        prefs->removeSrcFile = 0;
+    }
+
+    if (fCtx->hasStdoutOutput) return 0;
+    if (prefs->overwrite) return 0;
+
+    /* multiple files concatenated into single destination file using -o without -f */
+    if (g_display_prefs.displayLevel <= displayLevelCutoff) {
+        /* quiet mode => no prompt => fail automatically */
+        DISPLAYLEVEL(1, "Concatenating multiple processed inputs into a single output loses file metadata. \n");
+        DISPLAYLEVEL(1, "Aborting. \n");
+        return 1;
+    }
+    /* normal mode => prompt */
+    return UTIL_requireUserConfirmation("Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput);
+}
+
+static ZSTD_inBuffer setInBuffer(const void* buf, size_t s, size_t pos)
+{
+    ZSTD_inBuffer i;
+    i.src = buf;
+    i.size = s;
+    i.pos = pos;
+    return i;
+}
+
+static ZSTD_outBuffer setOutBuffer(void* buf, size_t s, size_t pos)
+{
+    ZSTD_outBuffer o;
+    o.dst = buf;
+    o.size = s;
+    o.pos = pos;
+    return o;
 }
 
 #ifndef ZSTD_NOCOMPRESS
@@ -856,9 +969,11 @@ typedef struct {
     void* dictBuffer;
     size_t dictBufferSize;
     const char* dictFileName;
+    stat_t dictFileStat;
     ZSTD_CStream* cctx;
     WritePoolCtx_t *writeCtx;
     ReadPoolCtx_t *readCtx;
+    int mmapDict;
 } cRess_t;
 
 /** ZSTD_cycleLog() :
@@ -899,11 +1014,14 @@ static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
 static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
                                     const char* dictFileName, unsigned long long const maxSrcFileSize,
                                     int cLevel, ZSTD_compressionParameters comprParams) {
+    U64 const dictSize = UTIL_getFileSize(dictFileName);
+    int const mmapDict = prefs->patchFromMode && PLATFORM_POSIX_VERSION < 1 && dictSize > prefs->memLimit;
     cRess_t ress;
     memset(&ress, 0, sizeof(ress));
 
     DISPLAYLEVEL(6, "FIO_createCResources \n");
     ress.cctx = ZSTD_createCCtx();
+    ress.mmapDict = mmapDict;
     if (ress.cctx == NULL)
         EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx",
                     strerror(errno));
@@ -912,9 +1030,14 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
      * because of memLimit check inside it */
     if (prefs->patchFromMode) {
         unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
-        FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
+        FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
+    }
+
+    if (!mmapDict) {
+        ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs, &ress.dictFileStat);   /* works with dictFileName==NULL */
+    } else {
+        ress.dictBufferSize = FIO_createDictBufferMMap(&ress.dictBuffer, dictFileName, prefs, &ress.dictFileStat);
     }
-    ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs);   /* works with dictFileName==NULL */
 
     ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
     ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
@@ -980,7 +1103,11 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
 
 static void FIO_freeCResources(const cRess_t* const ress)
 {
-    free(ress->dictBuffer);
+    if (!ress->mmapDict) {
+        free(ress->dictBuffer);
+    } else {
+        munmap(ress->dictBuffer, ress->dictBufferSize);
+    }
     AIO_WritePool_free(ress->writeCtx);
     AIO_ReadPool_free(ress->readCtx);
     ZSTD_freeCStream(ress->cctx);   /* never fails */
@@ -1006,7 +1133,7 @@ FIO_compressGzFrame(const cRess_t* ress,  /* buffers & handlers are used, but no
 
     {   int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
                         15 /* maxWindowLogSize */ + 16 /* gzip only */,
-                        8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */
+                        8, Z_DEFAULT_STRATEGY); /* see https://www.zlib.net/manual.html */
         if (ret != Z_OK) {
             EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
     }   }
@@ -1044,14 +1171,16 @@ FIO_compressGzFrame(const cRess_t* ress,  /* buffers & handlers are used, but no
                 strm.avail_out = (uInt)writeJob->bufferSize;
             }   }
         if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
-            DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
-                          (unsigned)(inFileSize>>20),
-                          (double)outFileSize/inFileSize*100)
+            DISPLAYUPDATE_PROGRESS(
+                    "\rRead : %u MB ==> %.2f%% ",
+                    (unsigned)(inFileSize>>20),
+                    (double)outFileSize/(double)inFileSize*100)
         } else {
-            DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
-                          (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
-                          (double)outFileSize/inFileSize*100);
-        }   }
+            DISPLAYUPDATE_PROGRESS(
+                    "\rRead : %u / %u MB ==> %.2f%% ",
+                    (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
+                    (double)outFileSize/(double)inFileSize*100);
+    }   }
 
     while (1) {
         int const ret = deflate(&strm, Z_FINISH);
@@ -1141,13 +1270,13 @@ FIO_compressLzmaFrame(cRess_t* ress,
                 strm.avail_out = writeJob->bufferSize;
         }   }
         if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
-            DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
+            DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
                             (unsigned)(inFileSize>>20),
-                            (double)outFileSize/inFileSize*100)
+                            (double)outFileSize/(double)inFileSize*100)
         else
-            DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
+            DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
                             (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
-                            (double)outFileSize/inFileSize*100);
+                            (double)outFileSize/(double)inFileSize*100);
         if (ret == LZMA_STREAM_END) break;
     }
 
@@ -1225,13 +1354,13 @@ FIO_compressLz4Frame(cRess_t* ress,
                             srcFileName, LZ4F_getErrorName(outSize));
             outFileSize += outSize;
             if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
-                DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
+                DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
                                 (unsigned)(inFileSize>>20),
-                                (double)outFileSize/inFileSize*100)
+                                (double)outFileSize/(double)inFileSize*100)
             } else {
-                DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
+                DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
                                 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
-                                (double)outFileSize/inFileSize*100);
+                                (double)outFileSize/(double)inFileSize*100);
             }
 
             /* Write Block */
@@ -1263,7 +1392,6 @@ FIO_compressLz4Frame(cRess_t* ress,
 }
 #endif
 
-
 static unsigned long long
 FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
                       FIO_prefs_t* const prefs,
@@ -1287,6 +1415,9 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
     unsigned inputPresented = 0;
     unsigned inputBlocked = 0;
     unsigned lastJobID = 0;
+    UTIL_time_t lastAdaptTime = UTIL_getTime();
+    U64 const adaptEveryMicro = REFRESH_RATE;
+
     UTIL_HumanReadableSize_t const file_hrs = UTIL_makeHumanReadableSize(fileSize);
 
     DISPLAYLEVEL(6, "compression using zstd format \n");
@@ -1324,7 +1455,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
         size_t stillToFlush;
         /* Fill input Buffer */
         size_t const inSize = AIO_ReadPool_fillBuffer(ress.readCtx, ZSTD_CStreamInSize());
-        ZSTD_inBuffer inBuff = { ress.readCtx->srcBuffer, ress.readCtx->srcBufferLoaded, 0 };
+        ZSTD_inBuffer inBuff = setInBuffer( ress.readCtx->srcBuffer, ress.readCtx->srcBufferLoaded, 0 );
         DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize);
         *readsize += inSize;
 
@@ -1336,7 +1467,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
             || (directive == ZSTD_e_end && stillToFlush != 0) ) {
 
             size_t const oldIPos = inBuff.pos;
-            ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 };
+            ZSTD_outBuffer outBuff = setOutBuffer( writeJob->buffer, writeJob->bufferSize, 0 );
             size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx);
             CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive));
             AIO_ReadPool_consumeBytes(ress.readCtx, inBuff.pos - oldIPos);
@@ -1355,131 +1486,137 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
                 compressedfilesize += outBuff.pos;
             }
 
-            /* display notification; and adapt compression level */
-            if (READY_FOR_UPDATE()) {
+            /* adaptive mode : statistics measurement and speed correction */
+            if (prefs->adaptiveMode && UTIL_clockSpanMicro(lastAdaptTime) > adaptEveryMicro) {
+                ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
+
+                lastAdaptTime = UTIL_getTime();
+
+                /* check output speed */
+                if (zfp.currentJobID > 1) {  /* only possible if nbWorkers >= 1 */
+
+                    unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced;
+                    unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed;
+                    assert(zfp.produced >= previous_zfp_update.produced);
+                    assert(prefs->nbWorkers >= 1);
+
+                    /* test if compression is blocked
+                        * either because output is slow and all buffers are full
+                        * or because input is slow and no job can start while waiting for at least one buffer to be filled.
+                        * note : exclude starting part, since currentJobID > 1 */
+                    if ( (zfp.consumed == previous_zfp_update.consumed)   /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/
+                        && (zfp.nbActiveWorkers == 0)                       /* confirmed : no compression ongoing */
+                        ) {
+                        DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
+                        speedChange = slower;
+                    }
+
+                    previous_zfp_update = zfp;
+
+                    if ( (newlyProduced > (newlyFlushed * 9 / 8))   /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
+                        && (flushWaiting == 0)                        /* flush speed was never slowed by lack of production, so it's operating at max capacity */
+                        ) {
+                        DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed);
+                        speedChange = slower;
+                    }
+                    flushWaiting = 0;
+                }
+
+                /* course correct only if there is at least one new job completed */
+                if (zfp.currentJobID > lastJobID) {
+                    DISPLAYLEVEL(6, "compression level adaptation check \n")
+
+                    /* check input speed */
+                    if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) {   /* warm up period, to fill all workers */
+                        if (inputBlocked <= 0) {
+                            DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
+                            speedChange = slower;
+                        } else if (speedChange == noChange) {
+                            unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested;
+                            unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed;
+                            unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced;
+                            unsigned long long newlyFlushed  = zfp.flushed  - previous_zfp_correction.flushed;
+                            previous_zfp_correction = zfp;
+                            assert(inputPresented > 0);
+                            DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
+                                            inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
+                                            (unsigned)newlyIngested, (unsigned)newlyConsumed,
+                                            (unsigned)newlyFlushed, (unsigned)newlyProduced);
+                            if ( (inputBlocked > inputPresented / 8)     /* input is waiting often, because input buffers is full : compression or output too slow */
+                                && (newlyFlushed * 33 / 32 > newlyProduced)  /* flush everything that is produced */
+                                && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
+                            ) {
+                                DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n",
+                                                newlyIngested, newlyConsumed, newlyProduced, newlyFlushed);
+                                speedChange = faster;
+                            }
+                        }
+                        inputBlocked = 0;
+                        inputPresented = 0;
+                    }
+
+                    if (speedChange == slower) {
+                        DISPLAYLEVEL(6, "slower speed , higher compression \n")
+                        compressionLevel ++;
+                        if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
+                        if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel;
+                        compressionLevel += (compressionLevel == 0);   /* skip 0 */
+                        ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
+                    }
+                    if (speedChange == faster) {
+                        DISPLAYLEVEL(6, "faster speed , lighter compression \n")
+                        compressionLevel --;
+                        if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel;
+                        compressionLevel -= (compressionLevel == 0);   /* skip 0 */
+                        ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
+                    }
+                    speedChange = noChange;
+
+                    lastJobID = zfp.currentJobID;
+                }  /* if (zfp.currentJobID > lastJobID) */
+            } /* if (prefs->adaptiveMode && UTIL_clockSpanMicro(lastAdaptTime) > adaptEveryMicro) */
+
+            /* display notification */
+            if (SHOULD_DISPLAY_PROGRESS() && READY_FOR_UPDATE()) {
                 ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
                 double const cShare = (double)zfp.produced / (double)(zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
                 UTIL_HumanReadableSize_t const buffered_hrs = UTIL_makeHumanReadableSize(zfp.ingested - zfp.consumed);
                 UTIL_HumanReadableSize_t const consumed_hrs = UTIL_makeHumanReadableSize(zfp.consumed);
                 UTIL_HumanReadableSize_t const produced_hrs = UTIL_makeHumanReadableSize(zfp.produced);
 
+                DELAY_NEXT_UPDATE();
+
                 /* display progress notifications */
+                DISPLAY_PROGRESS("\r%79s\r", "");    /* Clear out the current displayed line */
                 if (g_display_prefs.displayLevel >= 3) {
-                    DISPLAYUPDATE(3, "\r(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
-                                compressionLevel,
-                                buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
-                                consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
-                                produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
-                                cShare );
-                } else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) {
+                    /* Verbose progress update */
+                    DISPLAY_PROGRESS(
+                            "(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
+                            compressionLevel,
+                            buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
+                            consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
+                            produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
+                            cShare );
+                } else {
                     /* Require level 2 or forcibly displayed progress counter for summarized updates */
-                    DISPLAYLEVEL(1, "\r%79s\r", "");    /* Clear out the current displayed line */
                     if (fCtx->nbFilesTotal > 1) {
                         size_t srcFileNameSize = strlen(srcFileName);
                         /* Ensure that the string we print is roughly the same size each time */
                         if (srcFileNameSize > 18) {
                             const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
-                            DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ",
+                            DISPLAY_PROGRESS("Compress: %u/%u files. Current: ...%s ",
                                         fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName);
                         } else {
-                            DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ",
+                            DISPLAY_PROGRESS("Compress: %u/%u files. Current: %*s ",
                                         fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName);
                         }
                     }
-                    DISPLAYLEVEL(1, "Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
+                    DISPLAY_PROGRESS("Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
                     if (fileSize != UTIL_FILESIZE_UNKNOWN)
-                        DISPLAYLEVEL(2, "/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
-                    DISPLAYLEVEL(1, " ==> %2.f%%", cShare);
-                    DELAY_NEXT_UPDATE();
+                        DISPLAY_PROGRESS("/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
+                    DISPLAY_PROGRESS(" ==> %2.f%%", cShare);
                 }
-
-                /* adaptive mode : statistics measurement and speed correction */
-                if (prefs->adaptiveMode) {
-
-                    /* check output speed */
-                    if (zfp.currentJobID > 1) {  /* only possible if nbWorkers >= 1 */
-
-                        unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced;
-                        unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed;
-                        assert(zfp.produced >= previous_zfp_update.produced);
-                        assert(prefs->nbWorkers >= 1);
-
-                        /* test if compression is blocked
-                         * either because output is slow and all buffers are full
-                         * or because input is slow and no job can start while waiting for at least one buffer to be filled.
-                         * note : exclude starting part, since currentJobID > 1 */
-                        if ( (zfp.consumed == previous_zfp_update.consumed)   /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/
-                          && (zfp.nbActiveWorkers == 0)                       /* confirmed : no compression ongoing */
-                          ) {
-                            DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
-                            speedChange = slower;
-                        }
-
-                        previous_zfp_update = zfp;
-
-                        if ( (newlyProduced > (newlyFlushed * 9 / 8))   /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
-                          && (flushWaiting == 0)                        /* flush speed was never slowed by lack of production, so it's operating at max capacity */
-                          ) {
-                            DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed);
-                            speedChange = slower;
-                        }
-                        flushWaiting = 0;
-                    }
-
-                    /* course correct only if there is at least one new job completed */
-                    if (zfp.currentJobID > lastJobID) {
-                        DISPLAYLEVEL(6, "compression level adaptation check \n")
-
-                        /* check input speed */
-                        if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) {   /* warm up period, to fill all workers */
-                            if (inputBlocked <= 0) {
-                                DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
-                                speedChange = slower;
-                            } else if (speedChange == noChange) {
-                                unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested;
-                                unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed;
-                                unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced;
-                                unsigned long long newlyFlushed  = zfp.flushed  - previous_zfp_correction.flushed;
-                                previous_zfp_correction = zfp;
-                                assert(inputPresented > 0);
-                                DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
-                                                inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
-                                                (unsigned)newlyIngested, (unsigned)newlyConsumed,
-                                                (unsigned)newlyFlushed, (unsigned)newlyProduced);
-                                if ( (inputBlocked > inputPresented / 8)     /* input is waiting often, because input buffers is full : compression or output too slow */
-                                  && (newlyFlushed * 33 / 32 > newlyProduced)  /* flush everything that is produced */
-                                  && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
-                                ) {
-                                    DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n",
-                                                    newlyIngested, newlyConsumed, newlyProduced, newlyFlushed);
-                                    speedChange = faster;
-                                }
-                            }
-                            inputBlocked = 0;
-                            inputPresented = 0;
-                        }
-
-                        if (speedChange == slower) {
-                            DISPLAYLEVEL(6, "slower speed , higher compression \n")
-                            compressionLevel ++;
-                            if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
-                            if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel;
-                            compressionLevel += (compressionLevel == 0);   /* skip 0 */
-                            ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
-                        }
-                        if (speedChange == faster) {
-                            DISPLAYLEVEL(6, "faster speed , lighter compression \n")
-                            compressionLevel --;
-                            if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel;
-                            compressionLevel -= (compressionLevel == 0);   /* skip 0 */
-                            ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
-                        }
-                        speedChange = noChange;
-
-                        lastJobID = zfp.currentJobID;
-                    }  /* if (zfp.currentJobID > lastJobID) */
-                }  /* if (g_adaptiveMode) */
-            }  /* if (READY_FOR_UPDATE()) */
+            }  /* if (SHOULD_DISPLAY_PROGRESS() && READY_FOR_UPDATE()) */
         }  /* while ((inBuff.pos != inBuff.size) */
     } while (directive != ZSTD_e_end);
 
@@ -1555,20 +1692,18 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
     /* Status */
     fCtx->totalBytesInput += (size_t)readsize;
     fCtx->totalBytesOutput += (size_t)compressedfilesize;
-    DISPLAYLEVEL(2, "\r%79s\r", "");
-    if (g_display_prefs.displayLevel >= 2 &&
-        !fCtx->hasStdoutOutput &&
-        (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) {
+    DISPLAY_PROGRESS("\r%79s\r", "");
+    if (FIO_shouldDisplayFileSummary(fCtx)) {
         UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize);
         UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize);
         if (readsize == 0) {
-            DISPLAYLEVEL(2,"%-20s :  (%6.*f%s => %6.*f%s, %s) \n",
+            DISPLAY_SUMMARY("%-20s :  (%6.*f%s => %6.*f%s, %s) \n",
                 srcFileName,
                 hr_isize.precision, hr_isize.value, hr_isize.suffix,
                 hr_osize.precision, hr_osize.value, hr_osize.suffix,
                 dstFileName);
         } else {
-            DISPLAYLEVEL(2,"%-20s :%6.2f%%   (%6.*f%s => %6.*f%s, %s) \n",
+            DISPLAY_SUMMARY("%-20s :%6.2f%%   (%6.*f%s => %6.*f%s, %s) \n",
                 srcFileName,
                 (double)compressedfilesize / (double)readsize * 100,
                 hr_isize.precision, hr_isize.value, hr_isize.suffix,
@@ -1604,27 +1739,27 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
                                         cRess_t ress,
                                         const char* dstFileName,
                                         const char* srcFileName,
+                                        const stat_t* srcFileStat,
                                         int compressionLevel)
 {
     int closeDstFile = 0;
     int result;
-    stat_t statbuf;
-    int transferMTime = 0;
+    int transferStat = 0;
     FILE *dstFile;
+
     assert(AIO_ReadPool_getFile(ress.readCtx) != NULL);
     if (AIO_WritePool_getFile(ress.writeCtx) == NULL) {
-        int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
+        int dstFileInitialPermissions = DEFAULT_FILE_PERMISSIONS;
         if ( strcmp (srcFileName, stdinmark)
           && strcmp (dstFileName, stdoutmark)
-          && UTIL_stat(srcFileName, &statbuf)
-          && UTIL_isRegularFileStat(&statbuf) ) {
-            dstFilePermissions = statbuf.st_mode;
-            transferMTime = 1;
+          && UTIL_isRegularFileStat(srcFileStat) ) {
+            transferStat = 1;
+            dstFileInitialPermissions = TEMPORARY_FILE_PERMISSIONS;
         }
 
         closeDstFile = 1;
         DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName);
-        dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions);
+        dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFileInitialPermissions);
         if (dstFile==NULL) return 1;  /* could not open dstFileName */
         AIO_WritePool_setFile(ress.writeCtx, dstFile);
         /* Must only be added after FIO_openDstFile() succeeds.
@@ -1644,8 +1779,8 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
             DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
             result=1;
         }
-        if (transferMTime) {
-            UTIL_utime(dstFileName, &statbuf);
+        if (transferStat) {
+            UTIL_setFileStat(dstFileName, srcFileStat);
         }
         if ( (result != 0)  /* operation failure */
           && strcmp(dstFileName, stdoutmark)  /* special case : don't remove() stdout */
@@ -1687,18 +1822,26 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
 {
     int result;
     FILE* srcFile;
+    stat_t srcFileStat;
+    U64 fileSize = UTIL_FILESIZE_UNKNOWN;
     DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName);
 
-    /* ensure src is not a directory */
-    if (UTIL_isDirectory(srcFileName)) {
-        DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
-        return 1;
-    }
+    if (strcmp(srcFileName, stdinmark)) {
+        if (UTIL_stat(srcFileName, &srcFileStat)) {
+            /* failure to stat at all is handled during opening */
 
-    /* ensure src is not the same as dict (if present) */
-    if (ress.dictFileName != NULL && UTIL_isSameFile(srcFileName, ress.dictFileName)) {
-        DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
-        return 1;
+            /* ensure src is not a directory */
+            if (UTIL_isDirectoryStat(&srcFileStat)) {
+                DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
+                return 1;
+            }
+
+            /* ensure src is not the same as dict (if present) */
+            if (ress.dictFileName != NULL && UTIL_isSameFileStat(srcFileName, ress.dictFileName, &srcFileStat, &ress.dictFileStat)) {
+                DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
+                return 1;
+            }
+        }
     }
 
     /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
@@ -1710,16 +1853,30 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
         return 0;
     }
 
-    srcFile = FIO_openSrcFile(prefs, srcFileName);
+    srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
     if (srcFile == NULL) return 1;   /* srcFile could not be opened */
 
+    /* Don't use AsyncIO for small files */
+    if (strcmp(srcFileName, stdinmark)) /* Stdin doesn't have stats */
+        fileSize = UTIL_getFileSizeStat(&srcFileStat);
+    if(fileSize != UTIL_FILESIZE_UNKNOWN && fileSize < ZSTD_BLOCKSIZE_MAX * 3) {
+        AIO_ReadPool_setAsync(ress.readCtx, 0);
+        AIO_WritePool_setAsync(ress.writeCtx, 0);
+    } else {
+        AIO_ReadPool_setAsync(ress.readCtx, 1);
+        AIO_WritePool_setAsync(ress.writeCtx, 1);
+    }
+
     AIO_ReadPool_setFile(ress.readCtx, srcFile);
-    result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
+    result = FIO_compressFilename_dstFile(
+            fCtx, prefs, ress,
+            dstFileName, srcFileName,
+            &srcFileStat, compressionLevel);
     AIO_ReadPool_closeFile(ress.readCtx);
 
-    if ( prefs->removeSrcFile   /* --rm */
-      && result == 0       /* success */
-      && strcmp(srcFileName, stdinmark)   /* exception : don't erase stdin */
+    if ( prefs->removeSrcFile  /* --rm */
+      && result == 0           /* success */
+      && strcmp(srcFileName, stdinmark)  /* exception : don't erase stdin */
       ) {
         /* We must clear the handler, since after this point calling it would
          * delete both the source and destination files.
@@ -1741,7 +1898,8 @@ checked_index(const char* options[], size_t length, size_t index) {
 
 #define INDEX(options, index) checked_index((options), sizeof(options)  / sizeof(char*), (size_t)(index))
 
-void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) {
+void FIO_displayCompressionParameters(const FIO_prefs_t* prefs)
+{
     static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION,
                                            LZMA_EXTENSION, LZ4_EXTENSION};
     static const char* sparseOptions[3] = {" --no-sparse", "", " --sparse"};
@@ -1870,7 +2028,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
     assert(outFileName != NULL || suffix != NULL);
     if (outFileName != NULL) {   /* output into a single destination (stdout typically) */
         FILE *dstFile;
-        if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
+        if (FIO_multiFilesConcatWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
             FIO_freeCResources(&ress);
             return 1;
         }
@@ -1917,16 +2075,23 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
             FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);
     }
 
-    if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) {
+    if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
         UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput);
         UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput);
 
-        DISPLAYLEVEL(2, "\r%79s\r", "");
-        DISPLAYLEVEL(2, "%3d files compressed :%.2f%%   (%6.*f%4s => %6.*f%4s)\n",
-                        fCtx->nbFilesProcessed,
-                        (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
-                        hr_isize.precision, hr_isize.value, hr_isize.suffix,
-                        hr_osize.precision, hr_osize.value, hr_osize.suffix);
+        DISPLAY_PROGRESS("\r%79s\r", "");
+        if (fCtx->totalBytesInput == 0) {
+            DISPLAY_SUMMARY("%3d files compressed : (%6.*f%4s => %6.*f%4s)\n",
+                            fCtx->nbFilesProcessed,
+                            hr_isize.precision, hr_isize.value, hr_isize.suffix,
+                            hr_osize.precision, hr_osize.value, hr_osize.suffix);
+        } else {
+            DISPLAY_SUMMARY("%3d files compressed : %.2f%% (%6.*f%4s => %6.*f%4s)\n",
+                            fCtx->nbFilesProcessed,
+                            (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
+                            hr_isize.precision, hr_isize.value, hr_isize.suffix,
+                            hr_osize.precision, hr_osize.value, hr_osize.suffix);
+        }
     }
 
     FIO_freeCResources(&ress);
@@ -1950,11 +2115,13 @@ typedef struct {
 
 static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
 {
+    U64 const dictSize = UTIL_getFileSize(dictFileName);
+    int const mmapDict = prefs->patchFromMode && PLATFORM_POSIX_VERSION < 1 && dictSize > prefs->memLimit;
     dRess_t ress;
     memset(&ress, 0, sizeof(ress));
 
     if (prefs->patchFromMode)
-        FIO_adjustMemLimitForPatchFromMode(prefs, UTIL_getFileSize(dictFileName), 0 /* just use the dict size */);
+        FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
 
     /* Allocation */
     ress.dctx = ZSTD_createDStream();
@@ -1965,9 +2132,23 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
 
     /* dictionary */
     {   void* dictBuffer;
-        size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs);
-        CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) );
-        free(dictBuffer);
+        stat_t statbuf;
+        size_t dictBufferSize;
+
+        if (!mmapDict) {
+            dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs, &statbuf);
+        } else {
+            dictBufferSize = FIO_createDictBufferMMap(&dictBuffer, dictFileName, prefs, &statbuf);
+        }
+
+        CHECK( ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
+        CHECK( ZSTD_DCtx_loadDictionary(ress.dctx, dictBuffer, dictBufferSize) );
+
+        if (!mmapDict) {
+            free(dictBuffer);
+        } else {
+            munmap(dictBuffer, dictBufferSize);
+        }
     }
 
     ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_DStreamOutSize());
@@ -2064,10 +2245,9 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
 
     /* Main decompression Loop */
     while (1) {
-        ZSTD_inBuffer  inBuff = { ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 };
-        ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 };
+        ZSTD_inBuffer  inBuff = setInBuffer( ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 );
+        ZSTD_outBuffer outBuff= setOutBuffer( writeJob->buffer, writeJob->bufferSize, 0 );
         size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
-        const int displayLevel = (g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2;
         UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize);
         if (ZSTD_isError(readSizeHint)) {
             DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
@@ -2085,14 +2265,15 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
             size_t srcFileNameSize = strlen(srcFileName);
             if (srcFileNameSize > 18) {
                 const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
-                DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s...    ",
-                              fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
+                DISPLAYUPDATE_PROGRESS(
+                        "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s...    ",
+                        fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
             } else {
-                DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %.*f%s...    ",
+                DISPLAYUPDATE_PROGRESS("\rDecompress: %2u/%2u files. Current: %s : %.*f%s...    ",
                             fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix);
             }
         } else {
-            DISPLAYUPDATE(displayLevel, "\r%-20.20s : %.*f%s...     ",
+            DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s...     ",
                             srcFileName, hrs.precision, hrs.value, hrs.suffix);
         }
 
@@ -2134,7 +2315,7 @@ FIO_decompressGzFrame(dRess_t* ress, const char* srcFileName)
     strm.opaque = Z_NULL;
     strm.next_in = 0;
     strm.avail_in = 0;
-    /* see http://www.zlib.net/manual.html */
+    /* see https://www.zlib.net/manual.html */
     if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK)
         return FIO_ERROR_FRAME_DECODING;
 
@@ -2307,7 +2488,7 @@ FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName)
                 AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
                 filesize += decodedBytes;
                 hrs = UTIL_makeHumanReadableSize(filesize);
-                DISPLAYUPDATE(2, "\rDecompressed : %.*f%s  ", hrs.precision, hrs.value, hrs.suffix);
+                DISPLAYUPDATE_PROGRESS("\rDecompressed : %.*f%s  ", hrs.precision, hrs.value, hrs.suffix);
             }
 
             if (!nextToLoad) break;
@@ -2415,13 +2596,9 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
 
     /* Final Status */
     fCtx->totalBytesOutput += (size_t)filesize;
-    DISPLAYLEVEL(2, "\r%79s\r", "");
-    /* No status message in pipe mode (stdin - stdout) or multi-files mode */
-    if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) ||
-        g_display_prefs.displayLevel >= 3 ||
-        g_display_prefs.progressSetting == FIO_ps_always) {
-        DISPLAYLEVEL(1, "\r%-20s: %llu bytes \n", srcFileName, filesize);
-    }
+    DISPLAY_PROGRESS("\r%79s\r", "");
+    if (FIO_shouldDisplayFileSummary(fCtx))
+        DISPLAY_SUMMARY("%-20s: %llu bytes \n", srcFileName, filesize);
 
     return 0;
 }
@@ -2435,22 +2612,22 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
 static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
                                  FIO_prefs_t* const prefs,
                                  dRess_t ress,
-                                 const char* dstFileName, const char* srcFileName)
+                                 const char* dstFileName,
+                                 const char* srcFileName,
+                                 const stat_t* srcFileStat)
 {
     int result;
-    stat_t statbuf;
     int releaseDstFile = 0;
-    int transferMTime = 0;
+    int transferStat = 0;
 
     if ((AIO_WritePool_getFile(ress.writeCtx) == NULL) && (prefs->testMode == 0)) {
         FILE *dstFile;
         int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
         if ( strcmp(srcFileName, stdinmark)   /* special case : don't transfer permissions from stdin */
           && strcmp(dstFileName, stdoutmark)
-          && UTIL_stat(srcFileName, &statbuf)
-          && UTIL_isRegularFileStat(&statbuf) ) {
-            dstFilePermissions = statbuf.st_mode;
-            transferMTime = 1;
+          && UTIL_isRegularFileStat(srcFileStat) ) {
+            transferStat = 1;
+            dstFilePermissions = TEMPORARY_FILE_PERMISSIONS;
         }
 
         releaseDstFile = 1;
@@ -2475,8 +2652,8 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
             result = 1;
         }
 
-        if (transferMTime) {
-            UTIL_utime(dstFileName, &statbuf);
+        if (transferStat) {
+            UTIL_setFileStat(dstFileName, srcFileStat);
         }
 
         if ( (result != 0)  /* operation failure */
@@ -2498,18 +2675,32 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
 static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName)
 {
     FILE* srcFile;
+    stat_t srcFileStat;
     int result;
+    U64 fileSize = UTIL_FILESIZE_UNKNOWN;
 
     if (UTIL_isDirectory(srcFileName)) {
         DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
         return 1;
     }
 
-    srcFile = FIO_openSrcFile(prefs, srcFileName);
+    srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
     if (srcFile==NULL) return 1;
+
+    /* Don't use AsyncIO for small files */
+    if (strcmp(srcFileName, stdinmark)) /* Stdin doesn't have stats */
+        fileSize = UTIL_getFileSizeStat(&srcFileStat);
+    if(fileSize != UTIL_FILESIZE_UNKNOWN && fileSize < ZSTD_BLOCKSIZE_MAX * 3) {
+        AIO_ReadPool_setAsync(ress.readCtx, 0);
+        AIO_WritePool_setAsync(ress.writeCtx, 0);
+    } else {
+        AIO_ReadPool_setAsync(ress.readCtx, 1);
+        AIO_WritePool_setAsync(ress.writeCtx, 1);
+    }
+
     AIO_ReadPool_setFile(ress.readCtx, srcFile);
 
-    result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName);
+    result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName, &srcFileStat);
 
     AIO_ReadPool_setFile(ress.readCtx, NULL);
 
@@ -2686,7 +2877,7 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
     dRess_t ress = FIO_createDResources(prefs, dictFileName);
 
     if (outFileName) {
-        if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
+        if (FIO_multiFilesConcatWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
             FIO_freeDResources(ress);
             return 1;
         }
@@ -2730,8 +2921,11 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
             FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
     }
 
-    if (fCtx->nbFilesProcessed >= 1  && fCtx->nbFilesTotal > 1 && fCtx->totalBytesOutput != 0)
-        DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
+    if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
+        DISPLAY_PROGRESS("\r%79s\r", "");
+        DISPLAY_SUMMARY("%d files decompressed : %6llu bytes total \n",
+            fCtx->nbFilesProcessed, (unsigned long long)fCtx->totalBytesOutput);
+    }
 
     FIO_freeDResources(ress);
     return error;
@@ -2759,7 +2953,7 @@ typedef enum {
   info_frame_error=1,
   info_not_zstd=2,
   info_file_error=3,
-  info_truncated_input=4,
+  info_truncated_input=4
 } InfoError;
 
 #define ERROR_IF(c,n,...) {             \
@@ -2871,10 +3065,11 @@ static InfoError
 getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName)
 {
     InfoError status;
-    FILE* const srcFile = FIO_openSrcFile(NULL, inFileName);
+    stat_t srcFileStat;
+    FILE* const srcFile = FIO_openSrcFile(NULL, inFileName, &srcFileStat);
     ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName);
 
-    info->compressedSize = UTIL_getFileSize(inFileName);
+    info->compressedSize = UTIL_getFileSizeStat(&srcFileStat);
     status = FIO_analyzeFrames(info, srcFile);
 
     fclose(srcFile);
@@ -3010,7 +3205,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
     }   }
 
     if (numFiles == 0) {
-        if (!IS_CONSOLE(stdin)) {
+        if (!UTIL_isConsole(stdin)) {
             DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input \n");
         }
         DISPLAYLEVEL(1, "No files given \n");
diff --git a/programs/fileio.h b/programs/fileio.h
index b848934bc..291d4d414 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -86,7 +86,7 @@ void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch);
 void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit);
 void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers);
 void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog);
-void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
+void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, int flag);
 void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
 void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
 void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
diff --git a/programs/fileio_asyncio.c b/programs/fileio_asyncio.c
index 92c9a5b1d..fe9cca95d 100644
--- a/programs/fileio_asyncio.c
+++ b/programs/fileio_asyncio.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -140,7 +140,7 @@ int AIO_supported(void) {
 }
 
 /* ***********************************
- *  General IoPool implementation
+ *  Generic IoPool implementation
  *************************************/
 
 static IOJob_t *AIO_IOPool_createIoJob(IOPoolCtx_t *ctx, size_t bufferSize) {
@@ -163,20 +163,22 @@ static IOJob_t *AIO_IOPool_createIoJob(IOPoolCtx_t *ctx, size_t bufferSize) {
  * Displays warning if asyncio is requested but MT isn't available. */
 static void AIO_IOPool_createThreadPool(IOPoolCtx_t* ctx, const FIO_prefs_t* prefs) {
     ctx->threadPool = NULL;
+    ctx->threadPoolActive = 0;
     if(prefs->asyncIO) {
         if (ZSTD_pthread_mutex_init(&ctx->ioJobsMutex, NULL))
-            EXM_THROW(102,"Failed creating write availableJobs mutex");
+            EXM_THROW(102,"Failed creating ioJobsMutex mutex");
         /* We want MAX_IO_JOBS-2 queue items because we need to always have 1 free buffer to
          * decompress into and 1 buffer that's actively written to disk and owned by the writing thread. */
         assert(MAX_IO_JOBS >= 2);
         ctx->threadPool = POOL_create(1, MAX_IO_JOBS - 2);
+        ctx->threadPoolActive = 1;
         if (!ctx->threadPool)
-            EXM_THROW(104, "Failed creating writer thread pool");
+            EXM_THROW(104, "Failed creating I/O thread pool");
     }
 }
 
 /* AIO_IOPool_init:
- * Allocates and sets and a new write pool including its included availableJobs. */
+ * Allocates and sets and a new I/O thread pool including its included availableJobs. */
 static void AIO_IOPool_init(IOPoolCtx_t* ctx, const FIO_prefs_t* prefs, POOL_function poolFunction, size_t bufferSize) {
     int i;
     AIO_IOPool_createThreadPool(ctx, prefs);
@@ -192,27 +194,59 @@ static void AIO_IOPool_init(IOPoolCtx_t* ctx, const FIO_prefs_t* prefs, POOL_fun
 }
 
 
+/* AIO_IOPool_threadPoolActive:
+ * Check if current operation uses thread pool.
+ * Note that in some cases we have a thread pool initialized but choose not to use it. */
+static int AIO_IOPool_threadPoolActive(IOPoolCtx_t* ctx) {
+    return ctx->threadPool && ctx->threadPoolActive;
+}
+
+
+/* AIO_IOPool_lockJobsMutex:
+ * Locks the IO jobs mutex if threading is active */
+static void AIO_IOPool_lockJobsMutex(IOPoolCtx_t* ctx) {
+    if(AIO_IOPool_threadPoolActive(ctx))
+        ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex);
+}
+
+/* AIO_IOPool_unlockJobsMutex:
+ * Unlocks the IO jobs mutex if threading is active */
+static void AIO_IOPool_unlockJobsMutex(IOPoolCtx_t* ctx) {
+    if(AIO_IOPool_threadPoolActive(ctx))
+        ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex);
+}
+
 /* AIO_IOPool_releaseIoJob:
  * Releases an acquired job back to the pool. Doesn't execute the job. */
 static void AIO_IOPool_releaseIoJob(IOJob_t* job) {
     IOPoolCtx_t* const ctx = (IOPoolCtx_t *) job->ctx;
-    if(ctx->threadPool)
-        ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex);
+    AIO_IOPool_lockJobsMutex(ctx);
     assert(ctx->availableJobsCount < ctx->totalIoJobs);
     ctx->availableJobs[ctx->availableJobsCount++] = job;
-    if(ctx->threadPool)
-        ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex);
+    AIO_IOPool_unlockJobsMutex(ctx);
 }
 
 /* AIO_IOPool_join:
  * Waits for all tasks in the pool to finish executing. */
 static void AIO_IOPool_join(IOPoolCtx_t* ctx) {
-    if(ctx->threadPool)
+    if(AIO_IOPool_threadPoolActive(ctx))
         POOL_joinJobs(ctx->threadPool);
 }
 
+/* AIO_IOPool_setThreaded:
+ * Allows (de)activating threaded mode, to be used when the expected overhead
+ * of threading costs more than the expected gains. */
+static void AIO_IOPool_setThreaded(IOPoolCtx_t* ctx, int threaded) {
+    assert(threaded == 0 || threaded == 1);
+    assert(ctx != NULL);
+    if(ctx->threadPoolActive != threaded) {
+        AIO_IOPool_join(ctx);
+        ctx->threadPoolActive = threaded;
+    }
+}
+
 /* AIO_IOPool_free:
- * Release a previously allocated write thread pool. Makes sure all takss are done and released. */
+ * Release a previously allocated IO thread pool. Makes sure all tasks are done and released. */
 static void AIO_IOPool_destroy(IOPoolCtx_t* ctx) {
     int i;
     if(ctx->threadPool) {
@@ -236,12 +270,10 @@ static void AIO_IOPool_destroy(IOPoolCtx_t* ctx) {
 static IOJob_t* AIO_IOPool_acquireJob(IOPoolCtx_t* ctx) {
     IOJob_t *job;
     assert(ctx->file != NULL || ctx->prefs->testMode);
-    if(ctx->threadPool)
-        ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex);
+    AIO_IOPool_lockJobsMutex(ctx);
     assert(ctx->availableJobsCount > 0);
     job = (IOJob_t*) ctx->availableJobs[--ctx->availableJobsCount];
-    if(ctx->threadPool)
-        ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex);
+    AIO_IOPool_unlockJobsMutex(ctx);
     job->usedBufferSize = 0;
     job->file = ctx->file;
     job->offset = 0;
@@ -251,8 +283,7 @@ static IOJob_t* AIO_IOPool_acquireJob(IOPoolCtx_t* ctx) {
 
 /* AIO_IOPool_setFile:
  * Sets the destination file for future files in the pool.
- * Requires completion of all queues write jobs and release of all otherwise acquired jobs.
- * Also requires ending of sparse write if a previous file was used in sparse mode. */
+ * Requires completion of all queued jobs and release of all otherwise acquired jobs. */
 static void AIO_IOPool_setFile(IOPoolCtx_t* ctx, FILE* file) {
     assert(ctx!=NULL);
     AIO_IOPool_join(ctx);
@@ -269,7 +300,7 @@ static FILE* AIO_IOPool_getFile(const IOPoolCtx_t* ctx) {
  * The queued job shouldn't be used directly after queueing it. */
 static void AIO_IOPool_enqueueJob(IOJob_t* job) {
     IOPoolCtx_t* const ctx = (IOPoolCtx_t *)job->ctx;
-    if(ctx->threadPool)
+    if(AIO_IOPool_threadPoolActive(ctx))
         POOL_add(ctx->threadPool, ctx->poolFunction, job);
     else
         ctx->poolFunction(job);
@@ -300,8 +331,7 @@ void AIO_WritePool_enqueueAndReacquireWriteJob(IOJob_t **job) {
  * Blocks on completion of all current write jobs before executing. */
 void AIO_WritePool_sparseWriteEnd(WritePoolCtx_t* ctx) {
     assert(ctx != NULL);
-    if(ctx->base.threadPool)
-        POOL_joinJobs(ctx->base.threadPool);
+    AIO_IOPool_join(&ctx->base);
     AIO_fwriteSparseEnd(ctx->base.prefs, ctx->base.file, ctx->storedSkips);
     ctx->storedSkips = 0;
 }
@@ -368,6 +398,13 @@ void AIO_WritePool_free(WritePoolCtx_t* ctx) {
     free(ctx);
 }
 
+/* AIO_WritePool_setAsync:
+ * Allows (de)activating async mode, to be used when the expected overhead
+ * of asyncio costs more than the expected gains. */
+void AIO_WritePool_setAsync(WritePoolCtx_t* ctx, int async) {
+    AIO_IOPool_setThreaded(&ctx->base, async);
+}
+
 
 /* ***********************************
  *  ReadPool implementation
@@ -383,14 +420,13 @@ static void AIO_ReadPool_releaseAllCompletedJobs(ReadPoolCtx_t* ctx) {
 
 static void AIO_ReadPool_addJobToCompleted(IOJob_t* job) {
     ReadPoolCtx_t* const ctx = (ReadPoolCtx_t *)job->ctx;
-    if(ctx->base.threadPool)
-        ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex);
+    AIO_IOPool_lockJobsMutex(&ctx->base);
     assert(ctx->completedJobsCount < MAX_IO_JOBS);
     ctx->completedJobs[ctx->completedJobsCount++] = job;
-    if(ctx->base.threadPool) {
+    if(AIO_IOPool_threadPoolActive(&ctx->base)) {
         ZSTD_pthread_cond_signal(&ctx->jobCompletedCond);
-        ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex);
     }
+    AIO_IOPool_unlockJobsMutex(&ctx->base);
 }
 
 /* AIO_ReadPool_findNextWaitingOffsetCompletedJob_locked:
@@ -426,8 +462,7 @@ static size_t AIO_ReadPool_numReadsInFlight(ReadPoolCtx_t* ctx) {
  * Would block. */
 static IOJob_t* AIO_ReadPool_getNextCompletedJob(ReadPoolCtx_t* ctx) {
     IOJob_t *job = NULL;
-    if (ctx->base.threadPool)
-        ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex);
+    AIO_IOPool_lockJobsMutex(&ctx->base);
 
     job = AIO_ReadPool_findNextWaitingOffsetCompletedJob_locked(ctx);
 
@@ -443,8 +478,7 @@ static IOJob_t* AIO_ReadPool_getNextCompletedJob(ReadPoolCtx_t* ctx) {
         ctx->waitingOnOffset += job->usedBufferSize;
     }
 
-    if (ctx->base.threadPool)
-        ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex);
+    AIO_IOPool_unlockJobsMutex(&ctx->base);
     return job;
 }
 
@@ -524,7 +558,7 @@ ReadPoolCtx_t* AIO_ReadPool_create(const FIO_prefs_t* prefs, size_t bufferSize)
 
     if(ctx->base.threadPool)
         if (ZSTD_pthread_cond_init(&ctx->jobCompletedCond, NULL))
-            EXM_THROW(103,"Failed creating write jobCompletedCond mutex");
+            EXM_THROW(103,"Failed creating jobCompletedCond cond");
 
     return ctx;
 }
@@ -620,3 +654,10 @@ int AIO_ReadPool_closeFile(ReadPoolCtx_t* ctx) {
     AIO_ReadPool_setFile(ctx, NULL);
     return fclose(file);
 }
+
+/* AIO_ReadPool_setAsync:
+ * Allows (de)activating async mode, to be used when the expected overhead
+ * of asyncio costs more than the expected gains. */
+void AIO_ReadPool_setAsync(ReadPoolCtx_t* ctx, int async) {
+    AIO_IOPool_setThreaded(&ctx->base, async);
+}
diff --git a/programs/fileio_asyncio.h b/programs/fileio_asyncio.h
index 30db44b6e..feb25a3f9 100644
--- a/programs/fileio_asyncio.h
+++ b/programs/fileio_asyncio.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -8,6 +8,17 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
+ /*
+  * FileIO AsyncIO exposes read/write IO pools that allow doing IO asynchronously.
+  * Current implementation relies on having one thread that reads and one that
+  * writes.
+  * Each IO pool supports up to `MAX_IO_JOBS` that can be enqueued for work, but
+  * are performed serially by the appropriate worker thread.
+  * Most systems exposes better primitives to perform asynchronous IO, such as
+  * io_uring on newer linux systems. The API is built in such a way that in the
+  * future we could replace the threads with better solutions when available.
+  */
+
 #ifndef ZSTD_FILEIO_ASYNCIO_H
 #define ZSTD_FILEIO_ASYNCIO_H
 
@@ -27,6 +38,7 @@ extern "C" {
 typedef struct {
     /* These struct fields should be set only on creation and not changed afterwards */
     POOL_ctx* threadPool;
+    int threadPoolActive;
     int totalIoJobs;
     const FIO_prefs_t* prefs;
     POOL_function poolFunction;
@@ -136,6 +148,11 @@ WritePoolCtx_t* AIO_WritePool_create(const FIO_prefs_t* prefs, size_t bufferSize
  * Frees and releases a writePool and its resources. Closes destination file. */
 void AIO_WritePool_free(WritePoolCtx_t* ctx);
 
+/* AIO_WritePool_setAsync:
+ * Allows (de)activating async mode, to be used when the expected overhead
+ * of asyncio costs more than the expected gains. */
+void AIO_WritePool_setAsync(WritePoolCtx_t* ctx, int async);
+
 /* AIO_ReadPool_create:
  * Allocates and sets and a new readPool including its included jobs.
  * bufferSize should be set to the maximal buffer we want to read at a time, will also be used
@@ -146,6 +163,11 @@ ReadPoolCtx_t* AIO_ReadPool_create(const FIO_prefs_t* prefs, size_t bufferSize);
  * Frees and releases a readPool and its resources. Closes source file. */
 void AIO_ReadPool_free(ReadPoolCtx_t* ctx);
 
+/* AIO_ReadPool_setAsync:
+ * Allows (de)activating async mode, to be used when the expected overhead
+ * of asyncio costs more than the expected gains. */
+void AIO_ReadPool_setAsync(ReadPoolCtx_t* ctx, int async);
+
 /* AIO_ReadPool_consumeBytes:
  * Consumes byes from srcBuffer's beginning and updates srcBufferLoaded accordingly. */
 void AIO_ReadPool_consumeBytes(ReadPoolCtx_t *ctx, size_t n);
diff --git a/programs/fileio_common.h b/programs/fileio_common.h
index 282c2f13b..55491b8e3 100644
--- a/programs/fileio_common.h
+++ b/programs/fileio_common.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -38,16 +38,24 @@ extern FIO_display_prefs_t g_display_prefs;
 extern UTIL_time_t g_displayClock;
 
 #define REFRESH_RATE  ((U64)(SEC_TO_MICRO / 6))
-#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > REFRESH_RATE)
+#define READY_FOR_UPDATE() (UTIL_clockSpanMicro(g_displayClock) > REFRESH_RATE || g_display_prefs.displayLevel >= 4)
 #define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
 #define DISPLAYUPDATE(l, ...) {                              \
         if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \
-            if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \
+            if (READY_FOR_UPDATE()) { \
                 DELAY_NEXT_UPDATE();                         \
                 DISPLAY(__VA_ARGS__);                        \
                 if (g_display_prefs.displayLevel>=4) fflush(stderr);       \
     }   }   }
 
+#define SHOULD_DISPLAY_SUMMARY() \
+    (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always)
+#define SHOULD_DISPLAY_PROGRESS()                       \
+    (g_display_prefs.progressSetting != FIO_ps_never && SHOULD_DISPLAY_SUMMARY())
+#define DISPLAY_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYLEVEL(1, __VA_ARGS__); }}
+#define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }}
+#define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } }
+
 #undef MIN  /* in case it would be already defined */
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
 
@@ -114,4 +122,4 @@ extern UTIL_time_t g_displayClock;
 #if defined (__cplusplus)
 }
 #endif
-#endif //ZSTD_FILEIO_COMMON_H
+#endif /* ZSTD_FILEIO_COMMON_H */
diff --git a/programs/fileio_types.h b/programs/fileio_types.h
index a1fac2ca7..c1f42f1ad 100644
--- a/programs/fileio_types.h
+++ b/programs/fileio_types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/platform.h b/programs/platform.h
index b858e3b48..18a3587bf 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -33,7 +33,7 @@ extern "C" {
 
 /* **************************************
 *  Detect 64-bit OS
-*  http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
+*  https://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
 ****************************************/
 #if defined __ia64 || defined _M_IA64                                                                               /* Intel Itanium */ \
   || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__                                                /* POWER 64-bit */  \
@@ -80,7 +80,7 @@ extern "C" {
       * note: it's better to use unistd.h's _POSIX_VERSION whenever possible */
 #    define PLATFORM_POSIX_VERSION 200112L
 
-/* try to determine posix version through official unistd.h's _POSIX_VERSION (http://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
+/* try to determine posix version through official unistd.h's _POSIX_VERSION (https://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
  * note : there is no simple way to know in advance if  is present or not on target system,
  * Posix specification mandates its presence and its content, but target system must respect this spec.
  * It's necessary to _not_ #include  whenever target OS is not unix-like
@@ -127,6 +127,10 @@ extern "C" {
 
 /*-*********************************************
 *  Detect if isatty() and fileno() are available
+*
+*  Note: Use UTIL_isConsole() for the zstd CLI
+*  instead, as it allows faking is console for
+*  testing.
 ************************************************/
 #if (defined(__linux__) && (PLATFORM_POSIX_VERSION > 1)) \
  || (PLATFORM_POSIX_VERSION >= 200112L) \
@@ -192,13 +196,13 @@ static __inline int IS_CONSOLE(FILE* stdStream) {
 
 
 #ifndef ZSTD_SETPRIORITY_SUPPORT
-   /* mandates presence of  and support for setpriority() : http://man7.org/linux/man-pages/man2/setpriority.2.html */
+   /* mandates presence of  and support for setpriority() : https://man7.org/linux/man-pages/man2/setpriority.2.html */
 #  define ZSTD_SETPRIORITY_SUPPORT (PLATFORM_POSIX_VERSION >= 200112L)
 #endif
 
 
 #ifndef ZSTD_NANOSLEEP_SUPPORT
-   /* mandates support of nanosleep() within  : http://man7.org/linux/man-pages/man2/nanosleep.2.html */
+   /* mandates support of nanosleep() within  : https://man7.org/linux/man-pages/man2/nanosleep.2.html */
 #  if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) \
    || (PLATFORM_POSIX_VERSION >= 200112L)
 #     define ZSTD_NANOSLEEP_SUPPORT 1
diff --git a/programs/timefn.c b/programs/timefn.c
index 64577b0e9..f941e57e6 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -12,7 +12,8 @@
 /* ===  Dependencies  === */
 
 #include "timefn.h"
-
+#include "platform.h" /* set _POSIX_C_SOURCE */
+#include      /* CLOCK_MONOTONIC, TIME_UTC */
 
 /*-****************************************
 *  Time functions
@@ -20,12 +21,11 @@
 
 #if defined(_WIN32)   /* Windows */
 
+#include   /* LARGE_INTEGER */
 #include    /* abort */
 #include     /* perror */
 
-UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
-
-PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+UTIL_time_t UTIL_getTime(void)
 {
     static LARGE_INTEGER ticksPerSecond;
     static int init = 0;
@@ -36,30 +36,20 @@ PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
         }
         init = 1;
     }
-    return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
-}
-
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
-{
-    static LARGE_INTEGER ticksPerSecond;
-    static int init = 0;
-    if (!init) {
-        if (!QueryPerformanceFrequency(&ticksPerSecond)) {
-            perror("timefn::QueryPerformanceFrequency");
-            abort();
-        }
-        init = 1;
+    {   UTIL_time_t r;
+        LARGE_INTEGER x;
+        QueryPerformanceCounter(&x);
+        r.t = (PTime)(x.QuadPart * 1000000000ULL / ticksPerSecond.QuadPart);
+        return r;
     }
-    return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
 }
 
 
-
 #elif defined(__APPLE__) && defined(__MACH__)
 
-UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
+#include  /* mach_timebase_info_data_t, mach_timebase_info, mach_absolute_time */
 
-PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+UTIL_time_t UTIL_getTime(void)
 {
     static mach_timebase_info_data_t rate;
     static int init = 0;
@@ -67,23 +57,39 @@ PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
         mach_timebase_info(&rate);
         init = 1;
     }
-    return (((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom))/1000ULL;
-}
-
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
-{
-    static mach_timebase_info_data_t rate;
-    static int init = 0;
-    if (!init) {
-        mach_timebase_info(&rate);
-        init = 1;
+    {   UTIL_time_t r;
+        r.t = mach_absolute_time() * (PTime)rate.numer / (PTime)rate.denom;
+        return r;
+    }
+}
+
+/* POSIX.1-2001 (optional) */
+#elif defined(CLOCK_MONOTONIC)
+
+#include    /* abort */
+#include     /* perror */
+
+UTIL_time_t UTIL_getTime(void)
+{
+    /* time must be initialized, othersize it may fail msan test.
+     * No good reason, likely a limitation of timespec_get() for some target */
+    struct timespec time = { 0, 0 };
+    if (clock_gettime(CLOCK_MONOTONIC, &time) != 0) {
+        perror("timefn::clock_gettime(CLOCK_MONOTONIC)");
+        abort();
+    }
+    {   UTIL_time_t r;
+        r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
+        return r;
     }
-    return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom);
 }
 
 
-/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
-   Android also lacks it but does define TIME_UTC. */
+/* C11 requires support of timespec_get().
+ * However, FreeBSD 11 claims C11 compliance while lacking timespec_get().
+ * Double confirm timespec_get() support by checking the definition of TIME_UTC.
+ * However, some versions of Android manage to simultanously define TIME_UTC
+ * and lack timespec_get() support... */
 #elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
     && defined(TIME_UTC) && !defined(__ANDROID__)
 
@@ -94,65 +100,49 @@ UTIL_time_t UTIL_getTime(void)
 {
     /* time must be initialized, othersize it may fail msan test.
      * No good reason, likely a limitation of timespec_get() for some target */
-    UTIL_time_t time = UTIL_TIME_INITIALIZER;
+    struct timespec time = { 0, 0 };
     if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
-        perror("timefn::timespec_get");
+        perror("timefn::timespec_get(TIME_UTC)");
         abort();
     }
-    return time;
+    {   UTIL_time_t r;
+        r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
+        return r;
+    }
 }
 
-static UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
+
+#else   /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
+
+UTIL_time_t UTIL_getTime(void)
 {
-    UTIL_time_t diff;
-    if (end.tv_nsec < begin.tv_nsec) {
-        diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
-        diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
-    } else {
-        diff.tv_sec = end.tv_sec - begin.tv_sec;
-        diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
-    }
-    return diff;
+    UTIL_time_t r;
+    r.t = (PTime)clock() * 1000000000ULL / CLOCKS_PER_SEC;
+    return r;
+}
+
+#define TIME_MT_MEASUREMENTS_NOT_SUPPORTED
+
+#endif
+
+/* ==== Common functions, valid for all time API ==== */
+
+PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+    return clockEnd.t - clockStart.t;
 }
 
 PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
 {
-    UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
-    PTime micro = 0;
-    micro += 1000000ULL * diff.tv_sec;
-    micro += diff.tv_nsec / 1000ULL;
-    return micro;
+    return UTIL_getSpanTimeNano(begin, end) / 1000ULL;
 }
 
-PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
-{
-    UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
-    PTime nano = 0;
-    nano += 1000000000ULL * diff.tv_sec;
-    nano += diff.tv_nsec;
-    return nano;
-}
-
-
-
-#else   /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
-
-UTIL_time_t UTIL_getTime(void) { return clock(); }
-PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
-
-#endif
-
-
-
-/* returns time span in microseconds */
 PTime UTIL_clockSpanMicro(UTIL_time_t clockStart )
 {
     UTIL_time_t const clockEnd = UTIL_getTime();
     return UTIL_getSpanTimeMicro(clockStart, clockEnd);
 }
 
-/* returns time span in microseconds */
 PTime UTIL_clockSpanNano(UTIL_time_t clockStart )
 {
     UTIL_time_t const clockEnd = UTIL_getTime();
@@ -167,3 +157,12 @@ void UTIL_waitForNextTick(void)
         clockEnd = UTIL_getTime();
     } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
 }
+
+int UTIL_support_MT_measurements(void)
+{
+# if defined(TIME_MT_MEASUREMENTS_NOT_SUPPORTED)
+    return 0;
+# else
+    return 1;
+# endif
+}
diff --git a/programs/timefn.h b/programs/timefn.h
index 8ba8ed787..b814ff8d8 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,71 +16,51 @@ extern "C" {
 #endif
 
 
-/*-****************************************
-*  Dependencies
-******************************************/
-#include          /* clock_t, clock, CLOCKS_PER_SEC */
-
-
 
 /*-****************************************
-*  Local Types
+*  Types
 ******************************************/
 
 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 # if defined(_AIX)
 #  include 
 # else
-#  include  /* intptr_t */
+#  include  /* uint64_t */
 # endif
   typedef uint64_t           PTime;  /* Precise Time */
 #else
   typedef unsigned long long PTime;  /* does not support compilers without long long support */
 #endif
 
+/* UTIL_time_t contains a nanosecond time counter.
+ * The absolute value is not meaningful.
+ * It's only valid to compute the difference between 2 measurements. */
+typedef struct { PTime t; } UTIL_time_t;
+#define UTIL_TIME_INITIALIZER { 0 }
 
 
 /*-****************************************
 *  Time functions
 ******************************************/
-#if defined(_WIN32)   /* Windows */
-
-    #include    /* LARGE_INTEGER */
-    typedef LARGE_INTEGER UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER { { 0, 0 } }
-
-#elif defined(__APPLE__) && defined(__MACH__)
-
-    #include 
-    typedef PTime UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER 0
-
-/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
-   Android also lacks it but does define TIME_UTC. */
-#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
-    && defined(TIME_UTC) && !defined(__ANDROID__)
-
-    typedef struct timespec UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER { 0, 0 }
-
-#else   /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
-
-    #define UTIL_TIME_USES_C90_CLOCK
-    typedef clock_t UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER 0
-
-#endif
-
 
 UTIL_time_t UTIL_getTime(void);
-PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
 
-#define SEC_TO_MICRO ((PTime)1000000)
-PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
+/* Timer resolution can be low on some platforms.
+ * To improve accuracy, it's recommended to wait for a new tick
+ * before starting benchmark measurements */
+void UTIL_waitForNextTick(void);
+/* tells if timefn will return correct time measurements
+ * in presence of multi-threaded workload.
+ * note : this is not the case if only C90 clock_t measurements are available */
+int UTIL_support_MT_measurements(void);
+
+PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
 PTime UTIL_clockSpanNano(UTIL_time_t clockStart);
 
-void UTIL_waitForNextTick(void);
+PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
+PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
+
+#define SEC_TO_MICRO ((PTime)1000000)  /* nb of microseconds in a second */
 
 
 #if defined (__cplusplus)
diff --git a/programs/util.c b/programs/util.c
index 63b3ae176..e017772ef 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -66,6 +66,27 @@ extern "C" {
 #define UTIL_DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
 #define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
 
+static int g_traceDepth = 0;
+int g_traceFileStat = 0;
+
+#define UTIL_TRACE_CALL(...)                                         \
+    {                                                                \
+        if (g_traceFileStat) {                                       \
+            UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \
+            UTIL_DISPLAY(__VA_ARGS__);                               \
+            UTIL_DISPLAY("\n");                                      \
+            ++g_traceDepth;                                          \
+        }                                                            \
+    }
+
+#define UTIL_TRACE_RET(ret)                                                     \
+    {                                                                           \
+        if (g_traceFileStat) {                                                  \
+            --g_traceDepth;                                                     \
+            UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \
+        }                                                                      \
+    }
+
 /* A modified version of realloc().
  * If UTIL_realloc() fails the original block is freed.
  */
@@ -100,7 +121,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
     ch = getchar();
     result = 0;
     if (strchr(acceptableLetters, ch) == NULL) {
-        UTIL_DISPLAY("%s", abortMsg);
+        UTIL_DISPLAY("%s \n", abortMsg);
         result = 1;
     }
     /* flush the rest */
@@ -121,21 +142,34 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
 *  Functions
 ***************************************/
 
+void UTIL_traceFileStat(void)
+{
+    g_traceFileStat = 1;
+}
+
 int UTIL_stat(const char* filename, stat_t* statbuf)
 {
+    int ret;
+    UTIL_TRACE_CALL("UTIL_stat(%s)", filename);
 #if defined(_MSC_VER)
-    return !_stat64(filename, statbuf);
+    ret = !_stat64(filename, statbuf);
 #elif defined(__MINGW32__) && defined (__MSVCRT__)
-    return !_stati64(filename, statbuf);
+    ret = !_stati64(filename, statbuf);
 #else
-    return !stat(filename, statbuf);
+    ret = !stat(filename, statbuf);
 #endif
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 int UTIL_isRegularFile(const char* infilename)
 {
     stat_t statbuf;
-    return UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf);
+    int ret;
+    UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename);
+    ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf);
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 int UTIL_isRegularFileStat(const stat_t* statbuf)
@@ -151,71 +185,114 @@ int UTIL_isRegularFileStat(const stat_t* statbuf)
 int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
 {
     stat_t localStatBuf;
+    UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions);
     if (statbuf == NULL) {
-        if (!UTIL_stat(filename, &localStatBuf)) return 0;
+        if (!UTIL_stat(filename, &localStatBuf)) {
+            UTIL_TRACE_RET(0);
+            return 0;
+        }
         statbuf = &localStatBuf;
     }
-    if (!UTIL_isRegularFileStat(statbuf)) return 0; /* pretend success, but don't change anything */
-    return chmod(filename, permissions);
+    if (!UTIL_isRegularFileStat(statbuf)) {
+        UTIL_TRACE_RET(0);
+        return 0; /* pretend success, but don't change anything */
+    }
+    UTIL_TRACE_CALL("chmod");
+    {
+        int const ret = chmod(filename, permissions);
+        UTIL_TRACE_RET(ret);
+        UTIL_TRACE_RET(ret);
+        return ret;
+    }
 }
 
 /* set access and modification times */
 int UTIL_utime(const char* filename, const stat_t *statbuf)
 {
     int ret;
+    UTIL_TRACE_CALL("UTIL_utime(%s)", filename);
     /* We check that st_mtime is a macro here in order to give us confidence
      * that struct stat has a struct timespec st_mtim member. We need this
      * check because there are some platforms that claim to be POSIX 2008
      * compliant but which do not have st_mtim... */
 #if (PLATFORM_POSIX_VERSION >= 200809L) && defined(st_mtime)
-    /* (atime, mtime) */
-    struct timespec timebuf[2] = { {0, UTIME_NOW} };
-    timebuf[1] = statbuf->st_mtim;
-    ret = utimensat(AT_FDCWD, filename, timebuf, 0);
+    {
+        /* (atime, mtime) */
+        struct timespec timebuf[2] = { {0, UTIME_NOW} };
+        timebuf[1] = statbuf->st_mtim;
+        ret = utimensat(AT_FDCWD, filename, timebuf, 0);
+    }
 #else
-    struct utimbuf timebuf;
-    timebuf.actime = time(NULL);
-    timebuf.modtime = statbuf->st_mtime;
-    ret = utime(filename, &timebuf);
+    {
+        struct utimbuf timebuf;
+        timebuf.actime = time(NULL);
+        timebuf.modtime = statbuf->st_mtime;
+        ret = utime(filename, &timebuf);
+    }
 #endif
     errno = 0;
+    UTIL_TRACE_RET(ret);
     return ret;
 }
 
 int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
 {
     int res = 0;
-
     stat_t curStatBuf;
-    if (!UTIL_stat(filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf))
+    UTIL_TRACE_CALL("UTIL_setFileStat(%s)", filename);
+
+    if (!UTIL_stat(filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) {
+        UTIL_TRACE_RET(-1);
         return -1;
+    }
 
     /* set access and modification times */
     res += UTIL_utime(filename, statbuf);
 
+    /* Mimic gzip's behavior:
+     *
+     * "Change the group first, then the permissions, then the owner.
+     * That way, the permissions will be correct on systems that allow
+     * users to give away files, without introducing a security hole.
+     * Security depends on permissions not containing the setuid or
+     * setgid bits." */
+
 #if !defined(_WIN32)
-    res += chown(filename, statbuf->st_uid, statbuf->st_gid);  /* Copy ownership */
+    res += chown(filename, -1, statbuf->st_gid);  /* Apply group ownership */
 #endif
 
-    res += UTIL_chmod(filename, &curStatBuf, statbuf->st_mode & 07777);  /* Copy file permissions */
+    res += UTIL_chmod(filename, &curStatBuf, statbuf->st_mode & 0777);  /* Copy file permissions */
+
+#if !defined(_WIN32)
+    res += chown(filename, statbuf->st_uid, -1);  /* Apply user ownership */
+#endif
 
     errno = 0;
+    UTIL_TRACE_RET(-res);
     return -res; /* number of errors is returned */
 }
 
 int UTIL_isDirectory(const char* infilename)
 {
     stat_t statbuf;
-    return UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf);
+    int ret;
+    UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename);
+    ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf);
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 int UTIL_isDirectoryStat(const stat_t* statbuf)
 {
+    int ret;
+    UTIL_TRACE_CALL("UTIL_isDirectoryStat()");
 #if defined(_MSC_VER)
-    return (statbuf->st_mode & _S_IFDIR) != 0;
+    ret = (statbuf->st_mode & _S_IFDIR) != 0;
 #else
-    return S_ISDIR(statbuf->st_mode) != 0;
+    ret = S_ISDIR(statbuf->st_mode) != 0;
 #endif
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 int UTIL_compareStr(const void *p1, const void *p2) {
@@ -224,33 +301,68 @@ int UTIL_compareStr(const void *p1, const void *p2) {
 
 int UTIL_isSameFile(const char* fName1, const char* fName2)
 {
+    int ret;
     assert(fName1 != NULL); assert(fName2 != NULL);
+    UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2);
 #if defined(_MSC_VER) || defined(_WIN32)
     /* note : Visual does not support file identification by inode.
      *        inode does not work on Windows, even with a posix layer, like msys2.
      *        The following work-around is limited to detecting exact name repetition only,
      *        aka `filename` is considered different from `subdir/../filename` */
-    return !strcmp(fName1, fName2);
+    ret = !strcmp(fName1, fName2);
 #else
     {   stat_t file1Stat;
         stat_t file2Stat;
-        return UTIL_stat(fName1, &file1Stat)
+        ret =  UTIL_stat(fName1, &file1Stat)
             && UTIL_stat(fName2, &file2Stat)
-            && (file1Stat.st_dev == file2Stat.st_dev)
-            && (file1Stat.st_ino == file2Stat.st_ino);
+            && UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat);
     }
 #endif
+    UTIL_TRACE_RET(ret);
+    return ret;
+}
+
+int UTIL_isSameFileStat(
+        const char* fName1, const char* fName2,
+        const stat_t* file1Stat, const stat_t* file2Stat)
+{
+    int ret;
+    assert(fName1 != NULL); assert(fName2 != NULL);
+    UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2);
+#if defined(_MSC_VER) || defined(_WIN32)
+    /* note : Visual does not support file identification by inode.
+     *        inode does not work on Windows, even with a posix layer, like msys2.
+     *        The following work-around is limited to detecting exact name repetition only,
+     *        aka `filename` is considered different from `subdir/../filename` */
+    (void)file1Stat;
+    (void)file2Stat;
+    ret = !strcmp(fName1, fName2);
+#else
+    {
+        ret =  (file1Stat->st_dev == file2Stat->st_dev)
+            && (file1Stat->st_ino == file2Stat->st_ino);
+    }
+#endif
+    UTIL_TRACE_RET(ret);
+    return ret;
 }
 
 /* UTIL_isFIFO : distinguish named pipes */
 int UTIL_isFIFO(const char* infilename)
 {
+    UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename);
 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
 #if PLATFORM_POSIX_VERSION >= 200112L
-    stat_t statbuf;
-    if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) return 1;
+    {
+        stat_t statbuf;
+        if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) {
+            UTIL_TRACE_RET(1);
+            return 1;
+        }
+    }
 #endif
     (void)infilename;
+    UTIL_TRACE_RET(0);
     return 0;
 }
 
@@ -278,21 +390,69 @@ int UTIL_isBlockDevStat(const stat_t* statbuf)
 
 int UTIL_isLink(const char* infilename)
 {
+    UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename);
 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
 #if PLATFORM_POSIX_VERSION >= 200112L
-    stat_t statbuf;
-    int const r = lstat(infilename, &statbuf);
-    if (!r && S_ISLNK(statbuf.st_mode)) return 1;
+    {
+        stat_t statbuf;
+        int const r = lstat(infilename, &statbuf);
+        if (!r && S_ISLNK(statbuf.st_mode)) {
+            UTIL_TRACE_RET(1);
+            return 1;
+        }
+    }
 #endif
     (void)infilename;
+    UTIL_TRACE_RET(0);
     return 0;
 }
 
+static int g_fakeStdinIsConsole = 0;
+static int g_fakeStderrIsConsole = 0;
+static int g_fakeStdoutIsConsole = 0;
+
+int UTIL_isConsole(FILE* file)
+{
+    int ret;
+    UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file));
+    if (file == stdin && g_fakeStdinIsConsole)
+        ret = 1;
+    else if (file == stderr && g_fakeStderrIsConsole)
+        ret = 1;
+    else if (file == stdout && g_fakeStdoutIsConsole)
+        ret = 1;
+    else
+        ret = IS_CONSOLE(file);
+    UTIL_TRACE_RET(ret);
+    return ret;
+}
+
+void UTIL_fakeStdinIsConsole(void)
+{
+    g_fakeStdinIsConsole = 1;
+}
+void UTIL_fakeStdoutIsConsole(void)
+{
+    g_fakeStdoutIsConsole = 1;
+}
+void UTIL_fakeStderrIsConsole(void)
+{
+    g_fakeStderrIsConsole = 1;
+}
+
 U64 UTIL_getFileSize(const char* infilename)
 {
     stat_t statbuf;
-    if (!UTIL_stat(infilename, &statbuf)) return UTIL_FILESIZE_UNKNOWN;
-    return UTIL_getFileSizeStat(&statbuf);
+    UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename);
+    if (!UTIL_stat(infilename, &statbuf)) {
+        UTIL_TRACE_RET(-1);
+        return UTIL_FILESIZE_UNKNOWN;
+    }
+    {
+        U64 const size = UTIL_getFileSizeStat(&statbuf);
+        UTIL_TRACE_RET((int)size);
+        return size;
+    }
 }
 
 U64 UTIL_getFileSizeStat(const stat_t* statbuf)
@@ -369,11 +529,16 @@ U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles)
 {
     U64 total = 0;
     unsigned n;
+    UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles);
     for (n=0; n 200 MB/s per core, to strong modes with excellent compression ratios\. It also features a very fast decoder, with speeds > 500 MB/s per core\.
+\fBzstd\fR is a fast lossless compression algorithm and data compression tool, with command line syntax similar to \fBgzip\fR(1) and \fBxz\fR(1)\. It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages\. \fBzstd\fR offers highly configurable compression speed, from fast modes at > 200 MB/s per core, to strong modes with excellent compression ratios\. It also features a very fast decoder, with speeds > 500 MB/s per core\.
 .
 .P
-\fBzstd\fR command line syntax is generally similar to gzip, but features the following differences :
+\fBzstd\fR command line syntax is generally similar to gzip, but features the following differences:
 .
 .IP "\(bu" 4
 Source files are preserved by default\. It\'s possible to remove them automatically by using the \fB\-\-rm\fR command\.
@@ -34,10 +34,13 @@ When compressing a single file, \fBzstd\fR displays progress notifications and r
 .IP "\(bu" 4
 \fBzstd\fR does not accept input from console, though it does accept \fBstdin\fR when it\'s not the console\.
 .
+.IP "\(bu" 4
+\fBzstd\fR does not store the input\'s filename or attributes, only its contents\.
+.
 .IP "" 0
 .
 .P
-\fBzstd\fR processes each \fIfile\fR according to the selected operation mode\. If no \fIfiles\fR are given or \fIfile\fR is \fB\-\fR, \fBzstd\fR reads from standard input and writes the processed data to standard output\. \fBzstd\fR will refuse to write compressed data to standard output if it is a terminal : it will display an error message and skip the \fIfile\fR\. Similarly, \fBzstd\fR will refuse to read compressed data from standard input if it is a terminal\.
+\fBzstd\fR processes each \fIfile\fR according to the selected operation mode\. If no \fIfiles\fR are given or \fIfile\fR is \fB\-\fR, \fBzstd\fR reads from standard input and writes the processed data to standard output\. \fBzstd\fR will refuse to write compressed data to standard output if it is a terminal: it will display an error message and skip the file\. Similarly, \fBzstd\fR will refuse to read compressed data from standard input if it is a terminal\.
 .
 .P
 Unless \fB\-\-stdout\fR or \fB\-o\fR is specified, \fIfiles\fR are written to a new file whose name is derived from the source \fIfile\fR name:
@@ -50,12 +53,12 @@ When decompressing, the \fB\.zst\fR suffix is removed from the source filename t
 .
 .IP "" 0
 .
-.SS "Concatenation with \.zst files"
+.SS "Concatenation with \.zst Files"
 It is possible to concatenate multiple \fB\.zst\fR files\. \fBzstd\fR will decompress such agglomerated file as if it was a single \fB\.zst\fR file\.
 .
 .SH "OPTIONS"
 .
-.SS "Integer suffixes and special values"
+.SS "Integer Suffixes and Special Values"
 In most places where an integer argument is expected, an optional suffix is supported to easily indicate large integers\. There must be no space between the integer and the suffix\.
 .
 .TP
@@ -66,7 +69,7 @@ Multiply the integer by 1,024 (2^10)\. \fBKi\fR, \fBK\fR, and \fBKB\fR are accep
 \fBMiB\fR
 Multiply the integer by 1,048,576 (2^20)\. \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\.
 .
-.SS "Operation mode"
+.SS "Operation Mode"
 If multiple operation mode options are given, the last one takes effect\.
 .
 .TP
@@ -83,20 +86,20 @@ Test the integrity of compressed \fIfiles\fR\. This option is equivalent to \fB\
 .
 .TP
 \fB\-b#\fR
-Benchmark file(s) using compression level #
+Benchmark file(s) using compression level \fI#\fR\. See \fIBENCHMARK\fR below for a description of this operation\.
 .
 .TP
-\fB\-\-train FILEs\fR
-Use FILEs as a training set to create a dictionary\. The training set should contain a lot of small files (> 100)\.
+\fB\-\-train FILES\fR
+Use \fIFILES\fR as a training set to create a dictionary\. The training set should contain a lot of small files (> 100)\. See \fIDICTIONARY BUILDER\fR below for a description of this operation\.
 .
 .TP
 \fB\-l\fR, \fB\-\-list\fR
 Display information related to a zstd compressed file, such as size, ratio, and checksum\. Some of these fields may not be available\. This command\'s output can be augmented with the \fB\-v\fR modifier\.
 .
-.SS "Operation modifiers"
+.SS "Operation Modifiers"
 .
 .IP "\(bu" 4
-\fB\-#\fR: \fB#\fR compression level [1\-19] (default: 3)
+\fB\-#\fR: selects \fB#\fR compression level [1\-19] (default: 3)
 .
 .IP "\(bu" 4
 \fB\-\-ultra\fR: unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
@@ -108,13 +111,22 @@ Display information related to a zstd compressed file, such as size, ratio, and
 \fB\-T#\fR, \fB\-\-threads=#\fR: Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to \fBZSTDMT_NBWORKERS_MAX\fR, which is either 64 in 32\-bit mode, or 256 for 64\-bit environments\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
 .
 .IP "\(bu" 4
-\fB\-\-single\-thread\fR: Use a single thread for both I/O and compression\. As compression is serialized with I/O, this can be slightly slower\. Single\-thread mode features significantly lower memory usage, which can be useful for systems with limited amount of memory, such as 32\-bit systems\. Note 1 : this mode is the only available one when multithread support is disabled\. Note 2 : this mode is different from \fB\-T1\fR, which spawns 1 compression thread in parallel with I/O\. Final compressed result is also slightly different from \fB\-T1\fR\.
+\fB\-\-single\-thread\fR: Use a single thread for both I/O and compression\. As compression is serialized with I/O, this can be slightly slower\. Single\-thread mode features significantly lower memory usage, which can be useful for systems with limited amount of memory, such as 32\-bit systems\.
+.
+.IP
+Note 1: this mode is the only available one when multithread support is disabled\.
+.
+.IP
+Note 2: this mode is different from \fB\-T1\fR, which spawns 1 compression thread in parallel with I/O\. Final compressed result is also slightly different from \fB\-T1\fR\.
 .
 .IP "\(bu" 4
 \fB\-\-auto\-threads={physical,logical} (default: physical)\fR: When using a default amount of threads via \fB\-T0\fR, choose the default based on the number of detected physical or logical cores\.
 .
 .IP "\(bu" 4
-\fB\-\-adapt[=min=#,max=#]\fR : \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
+\fB\-\-adapt[=min=#,max=#]\fR: \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MiB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\.
+.
+.IP
+\fINote\fR: at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
 .
 .IP "\(bu" 4
 \fB\-\-long[=#]\fR: enables long distance matching with \fB#\fR \fBwindowLog\fR, if \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
@@ -126,40 +138,49 @@ Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \
 \fB\-D DICT\fR: use \fBDICT\fR as Dictionary to compress or decompress FILE(s)
 .
 .IP "\(bu" 4
-\fB\-\-patch\-from FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that windowSize > srcSize\.
+\fB\-\-patch\-from FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that \fIwindowSize\fR > \fIsrcSize\fR\.
 .
 .IP
-Note: cannot use both this and \-D together Note: \fB\-\-long\fR mode will be automatically activated if chainLog < fileLog (fileLog being the windowLog required to cover the whole file)\. You can also manually force it\. Note: for all levels, you can use \-\-patch\-from in \-\-single\-thread mode to improve compression ratio at the cost of speed Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e\. 4096), and by setting a large \fB\-\-zstd=chainLog=\fR
+Note: cannot use both this and \fB\-D\fR together\.
+.
+.IP
+Note: \fB\-\-long\fR mode will be automatically activated if \fIchainLog\fR < \fIfileLog\fR (\fIfileLog\fR being the \fIwindowLog\fR required to cover the whole file)\. You can also manually force it\.
+.
+.IP
+Note: for all levels, you can use \fB\-\-patch\-from\fR in \fB\-\-single\-thread\fR mode to improve compression ratio at the cost of speed\.
+.
+.IP
+Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e\. 4096), and by setting a large \fB\-\-zstd=chainLog=\fR\.
 .
 .IP "\(bu" 4
-\fB\-\-rsyncable\fR : \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your mileage may vary\.
+\fB\-\-rsyncable\fR: \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your mileage may vary\.
 .
 .IP "\(bu" 4
 \fB\-C\fR, \fB\-\-[no\-]check\fR: add integrity check computed from uncompressed data (default: enabled)
 .
 .IP "\(bu" 4
-\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \-\-content\-size (meaning that the original size will be placed in the header)\.
+\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \fB\-\-content\-size\fR (meaning that the original size will be placed in the header)\.
 .
 .IP "\(bu" 4
 \fB\-\-no\-dictID\fR: do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\.
 .
 .IP "\(bu" 4
-\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, \fBzstd\fR uses 128 MB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (i\.e\. you can increase or decrease it)\.
+\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, \fBzstd\fR uses 128 MiB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (i\.e\. you can increase or decrease it)\.
 .
 .IP
-This is also used during compression when using with \-\-patch\-from=\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MB)\.
+This is also used during compression when using with \fB\-\-patch\-from=\fR\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MiB)\.
 .
 .IP
-Additionally, this can be used to limit memory for dictionary training\. This parameter overrides the default limit of 2 GB\. zstd will load training samples up to the memory limit and ignore the rest\.
+Additionally, this can be used to limit memory for dictionary training\. This parameter overrides the default limit of 2 GiB\. zstd will load training samples up to the memory limit and ignore the rest\.
 .
 .IP "\(bu" 4
-\fB\-\-stream\-size=#\fR : Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
+\fB\-\-stream\-size=#\fR: Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
 .
 .IP "\(bu" 4
 \fB\-\-size\-hint=#\fR: When handling input from a stream, \fBzstd\fR must guess how large the source size will be when optimizing compression parameters\. If the stream size is relatively small, this guess may be a poor one, resulting in a higher compression ratio than expected\. This feature allows for controlling the guess when needed\. Exact guesses result in better compression ratios\. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation\.
 .
 .IP "\(bu" 4
-\fB\-o FILE\fR: save result into \fBFILE\fR
+\fB\-o FILE\fR: save result into \fBFILE\fR\.
 .
 .IP "\(bu" 4
 \fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\. During decompression and when the output destination is stdout, pass\-through unrecognized formats as\-is\.
@@ -171,10 +192,10 @@ Additionally, this can be used to limit memory for dictionary training\. This pa
 \fB\-\-[no\-]sparse\fR: enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\.
 .
 .IP "\(bu" 4
-\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\-f) option is set\.
+\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\fB\-f\fR) option is set\.
 .
 .IP "\(bu" 4
-\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \-o, will trigger a confirmation prompt (which can be silenced with \-f), as this is a destructive operation\.
+\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \fB\-o\fR, will trigger a confirmation prompt (which can be silenced with \fB\-f\fR), as this is a destructive operation\.
 .
 .IP "\(bu" 4
 \fB\-k\fR, \fB\-\-keep\fR: keep source file(s) after successful compression or decompression\. This is the default behavior\.
@@ -201,7 +222,7 @@ If input directory contains "\.\.", the files in this directory will be ignored\
 \fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR: display help/long help and exit
 .
 .IP "\(bu" 4
-\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced : \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\. \fB\-q\fR will only display the version number, suitable for machine reading\.
+\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced: \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\. \fB\-q\fR will only display the version number, suitable for machine reading\.
 .
 .IP "\(bu" 4
 \fB\-v\fR, \fB\-\-verbose\fR: verbose mode, display more information
@@ -213,14 +234,14 @@ If input directory contains "\.\.", the files in this directory will be ignored\
 \fB\-\-no\-progress\fR: do not display the progress bar, but keep all other messages\.
 .
 .IP "\(bu" 4
-\fB\-\-show\-default\-cparams\fR: Shows the default compression parameters that will be used for a particular src file\. If the provided src file is not a regular file (e\.g\. named pipe), the cli will just output the default parameters\. That is, the parameters that are used when the src size is unknown\.
+\fB\-\-show\-default\-cparams\fR: shows the default compression parameters that will be used for a particular input file, based on the provided compression level and the input size\. If the provided file is not a regular file (e\.g\. a pipe), this flag will output the parameters used for inputs of unknown size\.
 .
 .IP "\(bu" 4
 \fB\-\-\fR: All arguments after \fB\-\-\fR are treated as files
 .
 .IP "" 0
 .
-.SS "gzip Operation modifiers"
+.SS "gzip Operation Modifiers"
 When invoked via a \fBgzip\fR symlink, \fBzstd\fR will support further options that intend to mimic the \fBgzip\fR behavior:
 .
 .TP
@@ -231,7 +252,7 @@ do not store the original filename and timestamps when compressing a file\. This
 \fB\-\-best\fR
 alias to the option \fB\-9\fR\.
 .
-.SS "Interactions with Environment Variables"
+.SS "Environment Variables"
 Employing environment variables to set parameters has security implications\. Therefore, this avenue is intentionally limited\. Only \fBZSTD_CLEVEL\fR and \fBZSTD_NBTHREADS\fR are currently supported\. They set the compression level and number of threads to use during compression, respectively\.
 .
 .P
@@ -251,7 +272,7 @@ They can both be overridden by corresponding command line arguments: \fB\-#\fR f
 Use FILEs as training set to create a dictionary\. The training set should ideally contain a lot of samples (> 100), and weight typically 100x the target dictionary size (for example, ~10 MB for a 100 KB dictionary)\. \fB\-\-train\fR can be combined with \fB\-r\fR to indicate a directory rather than listing all the files, which can be useful to circumvent shell expansion limits\.
 .
 .IP
-Since dictionary compression is mostly effective for small files, the expectation is that the training set will only contain small files\. In the case where some samples happen to be large, only the first 128 KB of these samples will be used for training\.
+Since dictionary compression is mostly effective for small files, the expectation is that the training set will only contain small files\. In the case where some samples happen to be large, only the first 128 KiB of these samples will be used for training\.
 .
 .IP
 \fB\-\-train\fR supports multithreading if \fBzstd\fR is compiled with threading support (default)\. Additional advanced parameters can be specified with \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. The slower cover dictionary builder can be accessed with \fB\-\-train\-cover\fR\. Default \fB\-\-train\fR is equivalent to \fB\-\-train\-fastcover=d=8,steps=4\fR\.
@@ -281,7 +302,10 @@ In situations where the training set is larger than maximum memory, the CLI will
 .
 .TP
 \fB\-\-dictID=#\fR
-A dictionary ID is a locally unique ID\. The decoder will use this value to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to provide an explicit number ID instead\. It\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. Note that short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\.
+A dictionary ID is a locally unique ID\. The decoder will use this value to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to provide an explicit number ID instead\. It\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. Note that short numbers have an advantage: an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\.
+.
+.IP
+Note that RFC8878 reserves IDs less than 32768 and greater than or equal to 2^31, so they should not be used in public\.
 .
 .TP
 \fB\-\-train\-cover[=k#,d=#,steps=#,split=#,shrink[=#]]\fR
@@ -366,7 +390,7 @@ cut file(s) into independent chunks of size # (default: no chunking)
 set process priority to real\-time
 .
 .P
-\fBOutput Format:\fR CompressionLevel#Filename : InputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
+\fBOutput Format:\fR CompressionLevel#Filename: InputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
 .
 .P
 \fBMethodology:\fR For both compression and decompression speed, the entire input is compressed/decompressed in\-memory to measure speed\. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy\.
@@ -377,14 +401,14 @@ set process priority to real\-time
 Specify the size of each compression job\. This parameter is only available when multi\-threading is enabled\. Each compression job is run in parallel, so this value indirectly impacts the nb of active threads\. Default job size varies depending on compression level (generally \fB4 * windowSize\fR)\. \fB\-B#\fR makes it possible to manually select a custom size\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 512 KB, or \fBoverlapSize\fR, whichever is largest\. Different job sizes will lead to non\-identical compressed frames\.
 .
 .SS "\-\-zstd[=options]:"
-\fBzstd\fR provides 22 predefined compression levels\. The selected or default predefined compression level can be changed with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
+\fBzstd\fR provides 22 predefined regular compression levels plus the fast levels\. This compression level is translated internally into a number of specific parameters that actually control the behavior of the compressor\. (You can see the result of this translation with \fB\-\-show\-default\-cparams\fR\.) These specific parameters can be overridden with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
 .
 .TP
 \fBstrategy\fR=\fIstrat\fR, \fBstrat\fR=\fIstrat\fR
 Specify a strategy used by a match finder\.
 .
 .IP
-There are 9 strategies numbered from 1 to 9, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra, 9=ZSTD_btultra2\.
+There are 9 strategies numbered from 1 to 9, from fastest to strongest: 1=\fBZSTD_fast\fR, 2=\fBZSTD_dfast\fR, 3=\fBZSTD_greedy\fR, 4=\fBZSTD_lazy\fR, 5=\fBZSTD_lazy2\fR, 6=\fBZSTD_btlazy2\fR, 7=\fBZSTD_btopt\fR, 8=\fBZSTD_btultra\fR, 9=\fBZSTD_btultra2\fR\.
 .
 .TP
 \fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR
@@ -404,17 +428,17 @@ Specify the maximum number of bits for a hash table\.
 Bigger hash tables cause fewer collisions which usually makes compression faster, but requires more memory during compression\.
 .
 .IP
-The minimum \fIhlog\fR is 6 (64 B) and the maximum is 30 (1 GiB)\.
+The minimum \fIhlog\fR is 6 (64 entries / 256 B) and the maximum is 30 (1B entries / 4 GiB)\.
 .
 .TP
 \fBchainLog\fR=\fIclog\fR, \fBclog\fR=\fIclog\fR
-Specify the maximum number of bits for a hash chain or a binary tree\.
+Specify the maximum number of bits for the secondary search structure, whose form depends on the selected \fBstrategy\fR\.
 .
 .IP
-Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the ZSTD_fast strategy\.
+Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the \fBZSTD_fast\fR \fBstrategy\fR, which only has the primary hash table\.
 .
 .IP
-The minimum \fIclog\fR is 6 (64 B) and the maximum is 29 (524 Mib) on 32\-bit platforms and 30 (1 Gib) on 64\-bit platforms\.
+The minimum \fIclog\fR is 6 (64 entries / 256 B) and the maximum is 29 (512M entries / 2 GiB) on 32\-bit platforms and 30 (1B entries / 4 GiB) on 64\-bit platforms\.
 .
 .TP
 \fBsearchLog\fR=\fIslog\fR, \fBslog\fR=\fIslog\fR
@@ -441,20 +465,23 @@ The minimum \fImml\fR is 3 and the maximum is 7\.
 The impact of this field vary depending on selected strategy\.
 .
 .IP
-For ZSTD_btopt, ZSTD_btultra and ZSTD_btultra2, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLength\fR usually improves compression ratio but decreases compression speed\. t For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLength\fR increases compression speed but decreases compression ratio\.
+For \fBZSTD_btopt\fR, \fBZSTD_btultra\fR and \fBZSTD_btultra2\fR, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLength\fR usually improves compression ratio but decreases compression speed\.
+.
+.IP
+For \fBZSTD_fast\fR, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed: a larger \fBtargetLength\fR increases compression speed but decreases compression ratio\.
 .
 .IP
 For all other strategies, this field has no impact\.
 .
 .IP
-The minimum \fItlen\fR is 0 and the maximum is 128 Kib\.
+The minimum \fItlen\fR is 0 and the maximum is 128 KiB\.
 .
 .TP
 \fBoverlapLog\fR=\fIovlog\fR, \fBovlog\fR=\fIovlog\fR
 Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This parameter is only available when multithreading is enabled\. Reloading more data improves compression ratio, but decreases speed\.
 .
 .IP
-The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default" : \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
+The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default": \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
 .
 .TP
 \fBldmHashLog\fR=\fIlhlog\fR, \fBlhlog\fR=\fIlhlog\fR
@@ -514,6 +541,12 @@ The following parameters sets advanced compression options to something similar
 .P
 \fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
 .
+.SH "SEE ALSO"
+\fBzstdgrep\fR(1), \fBzstdless\fR(1), \fBgzip\fR(1), \fBxz\fR(1)
+.
+.P
+The \fIzstandard\fR format is specified in Y\. Collet, "Zstandard Compression and the \'application/zstd\' Media Type", https://www\.ietf\.org/rfc/rfc8878\.txt, Internet RFC 8878 (February 2021)\.
+.
 .SH "BUGS"
 Report bugs at: https://github\.com/facebook/zstd/issues
 .
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 37c2ba187..3b7f24f79 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -4,7 +4,7 @@ zstd(1) -- zstd, zstdmt, unzstd, zstdcat - Compress or decompress .zst files
 SYNOPSIS
 --------
 
-`zstd` [*OPTIONS*] [-|_INPUT-FILE_] [-o _OUTPUT-FILE_]
+`zstd` [] [-|] [-o ]
 
 `zstdmt` is equivalent to `zstd -T0`
 
@@ -16,7 +16,7 @@ SYNOPSIS
 DESCRIPTION
 -----------
 `zstd` is a fast lossless compression algorithm and data compression tool,
-with command line syntax similar to `gzip (1)` and `xz (1)`.
+with command line syntax similar to `gzip`(1) and `xz`(1).
 It is based on the **LZ77** family, with further FSE & huff0 entropy stages.
 `zstd` offers highly configurable compression speed,
 from fast modes at > 200 MB/s per core,
@@ -24,7 +24,7 @@ to strong modes with excellent compression ratios.
 It also features a very fast decoder, with speeds > 500 MB/s per core.
 
 `zstd` command line syntax is generally similar to gzip,
-but features the following differences :
+but features the following differences:
 
   - Source files are preserved by default.
     It's possible to remove them automatically by using the `--rm` command.
@@ -35,12 +35,13 @@ but features the following differences :
     Use `-q` to turn it off.
   - `zstd` does not accept input from console,
     though it does accept `stdin` when it's not the console.
+  - `zstd` does not store the input's filename or attributes, only its contents.
 
 `zstd` processes each _file_ according to the selected operation mode.
 If no _files_ are given or _file_ is `-`, `zstd` reads from standard input
 and writes the processed data to standard output.
 `zstd` will refuse to write compressed data to standard output
-if it is a terminal : it will display an error message and skip the _file_.
+if it is a terminal: it will display an error message and skip the file.
 Similarly, `zstd` will refuse to read compressed data from standard input
 if it is a terminal.
 
@@ -52,14 +53,15 @@ whose name is derived from the source _file_ name:
 * When decompressing, the `.zst` suffix is removed from the source filename to
   get the target filename
 
-### Concatenation with .zst files
+### Concatenation with .zst Files
 It is possible to concatenate multiple `.zst` files. `zstd` will decompress
 such agglomerated file as if it was a single `.zst` file.
 
 OPTIONS
 -------
 
-### Integer suffixes and special values
+### Integer Suffixes and Special Values
+
 In most places where an integer argument is expected,
 an optional suffix is supported to easily indicate large integers.
 There must be no space between the integer and the suffix.
@@ -71,7 +73,8 @@ There must be no space between the integer and the suffix.
     Multiply the integer by 1,048,576 (2\^20).
     `Mi`, `M`, and `MB` are accepted as synonyms for `MiB`.
 
-### Operation mode
+### Operation Mode
+
 If multiple operation mode options are given,
 the last one takes effect.
 
@@ -88,19 +91,21 @@ the last one takes effect.
     decompressed data is discarded and checksummed for errors.
     No files are created or removed.
 * `-b#`:
-    Benchmark file(s) using compression level #
-* `--train FILEs`:
-    Use FILEs as a training set to create a dictionary.
+    Benchmark file(s) using compression level _#_.
+    See _BENCHMARK_ below for a description of this operation.
+* `--train FILES`:
+    Use _FILES_ as a training set to create a dictionary.
     The training set should contain a lot of small files (> 100).
+    See _DICTIONARY BUILDER_ below for a description of this operation.
 * `-l`, `--list`:
     Display information related to a zstd compressed file, such as size, ratio, and checksum.
     Some of these fields may not be available.
     This command's output can be augmented with the `-v` modifier.
 
-### Operation modifiers
+### Operation Modifiers
 
 * `-#`:
-    `#` compression level \[1-19] (default: 3)
+    selects `#` compression level \[1-19\] (default: 3)
 * `--ultra`:
     unlocks high compression levels 20+ (maximum 22), using a lot more memory.
     Note that decompression will also require more memory when using these levels.
@@ -122,21 +127,24 @@ the last one takes effect.
     As compression is serialized with I/O, this can be slightly slower.
     Single-thread mode features significantly lower memory usage,
     which can be useful for systems with limited amount of memory, such as 32-bit systems.
-    Note 1 : this mode is the only available one when multithread support is disabled.
-    Note 2 : this mode is different from `-T1`, which spawns 1 compression thread in parallel with I/O.
+
+    Note 1: this mode is the only available one when multithread support is disabled.
+
+    Note 2: this mode is different from `-T1`, which spawns 1 compression thread in parallel with I/O.
     Final compressed result is also slightly different from `-T1`.
 * `--auto-threads={physical,logical} (default: physical)`:
     When using a default amount of threads via `-T0`, choose the default based on the number
     of detected physical or logical cores.
-* `--adapt[=min=#,max=#]` :
+* `--adapt[=min=#,max=#]`:
     `zstd` will dynamically adapt compression level to perceived I/O conditions.
     Compression level adaptation can be observed live by using command `-v`.
     Adaptation can be constrained between supplied `min` and `max` levels.
     The feature works when combined with multi-threading and `--long` mode.
     It does not work with `--single-thread`.
-    It sets window size to 8 MB by default (can be changed manually, see `wlog`).
+    It sets window size to 8 MiB by default (can be changed manually, see `wlog`).
     Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible.
-    _note_ : at the time of this writing, `--adapt` can remain stuck at low speed
+
+    _Note_: at the time of this writing, `--adapt` can remain stuck at low speed
     when combined with multiple worker threads (>=2).
 * `--long[=#]`:
     enables long distance matching with `#` `windowLog`, if `#` is not
@@ -153,18 +161,21 @@ the last one takes effect.
 * `--patch-from FILE`:
     Specify the file to be used as a reference point for zstd's diff engine.
     This is effectively dictionary compression with some convenient parameter
-    selection, namely that windowSize > srcSize.
+    selection, namely that _windowSize_ > _srcSize_.
 
-    Note: cannot use both this and -D together
-    Note: `--long` mode will be automatically activated if chainLog < fileLog
-        (fileLog being the windowLog required to cover the whole file). You
+    Note: cannot use both this and `-D` together.
+
+    Note: `--long` mode will be automatically activated if _chainLog_ < _fileLog_
+        (_fileLog_ being the _windowLog_ required to cover the whole file). You
         can also manually force it.
-    Note: for all levels, you can use --patch-from in --single-thread mode
-        to improve compression ratio at the cost of speed
+
+    Note: for all levels, you can use `--patch-from` in `--single-thread` mode
+        to improve compression ratio at the cost of speed.
+
     Note: for level 19, you can get increased compression ratio at the cost
         of speed by specifying `--zstd=targetLength=` to be something large
-        (i.e. 4096), and by setting a large `--zstd=chainLog=`
-* `--rsyncable` :
+        (i.e. 4096), and by setting a large `--zstd=chainLog=`.
+* `--rsyncable`:
     `zstd` will periodically synchronize the compression state to make the
     compressed file more rsync-friendly. There is a negligible impact to
     compression ratio, and the faster compression levels will see a small
@@ -177,24 +188,24 @@ the last one takes effect.
 * `--[no-]content-size`:
     enable / disable whether or not the original size of the file is placed in
     the header of the compressed file. The default option is
-    --content-size (meaning that the original size will be placed in the header).
+    `--content-size` (meaning that the original size will be placed in the header).
 * `--no-dictID`:
     do not store dictionary ID within frame header (dictionary compression).
     The decoder will have to rely on implicit knowledge about which dictionary to use,
     it won't be able to check if it's correct.
 * `-M#`, `--memory=#`:
-    Set a memory usage limit. By default, `zstd` uses 128 MB for decompression
+    Set a memory usage limit. By default, `zstd` uses 128 MiB for decompression
     as the maximum amount of memory the decompressor is allowed to use, but you can
     override this manually if need be in either direction (i.e. you can increase or
     decrease it).
 
-    This is also used during compression when using with --patch-from=. In this case,
-    this parameter overrides that maximum size allowed for a dictionary. (128 MB).
+    This is also used during compression when using with `--patch-from=`. In this case,
+    this parameter overrides that maximum size allowed for a dictionary. (128 MiB).
 
     Additionally, this can be used to limit memory for dictionary training. This parameter
-    overrides the default limit of 2 GB. zstd will load training samples up to the memory limit
+    overrides the default limit of 2 GiB. zstd will load training samples up to the memory limit
     and ignore the rest.
-* `--stream-size=#` :
+* `--stream-size=#`:
     Sets the pledged source size of input coming from a stream. This value must be exact, as it
     will be included in the produced frame header. Incorrect stream sizes will cause an error.
     This information will be used to better optimize compression parameters, resulting in
@@ -207,7 +218,7 @@ the last one takes effect.
     Exact guesses result in better compression ratios. Overestimates result in slightly
     degraded compression ratios, while underestimates may result in significant degradation.
 * `-o FILE`:
-    save result into `FILE`
+    save result into `FILE`.
 * `-f`, `--force`:
     disable input and output checks. Allows overwriting existing files, input
     from console, output to stdout, operating on links, block devices, etc.
@@ -227,11 +238,13 @@ the last one takes effect.
     enable / disable passing through uncompressed files as-is. During
     decompression when pass-through is enabled, unrecognized formats will be
     copied as-is from the input to the output. By default, pass-through will
-    occur when the output destination is stdout and the force (-f) option is
+    occur when the output destination is stdout and the force (`-f`) option is
     set.
 * `--rm`:
-    remove source file(s) after successful compression or decompression. If used in combination with
-    -o, will trigger a confirmation prompt (which can be silenced with -f), as this is a destructive operation.
+    remove source file(s) after successful compression or decompression.
+    This command is silently ignored if output is `stdout`.
+    If used in combination with `-o`,
+    triggers a confirmation prompt (which can be silenced with `-f`), as this is a destructive operation.
 * `-k`, `--keep`:
     keep source file(s) after successful compression or decompression.
     This is the default behavior.
@@ -270,7 +283,7 @@ the last one takes effect.
     display help/long help and exit
 * `-V`, `--version`:
     display version number and exit.
-    Advanced : `-vV` also displays supported formats.
+    Advanced: `-vV` also displays supported formats.
     `-vvV` also displays POSIX support.
     `-q` will only display the version number, suitable for machine reading.
 * `-v`, `--verbose`:
@@ -281,15 +294,13 @@ the last one takes effect.
 * `--no-progress`:
     do not display the progress bar, but keep all other messages.
 * `--show-default-cparams`:
-    Shows the default compression parameters that will be used for a
-    particular src file. If the provided src file is not a regular file
-    (e.g. named pipe), the cli will just output the default parameters.
-    That is, the parameters that are used when the src size is unknown.
+    shows the default compression parameters that will be used for a particular input file, based on the provided compression level and the input size.
+    If the provided file is not a regular file (e.g. a pipe), this flag will output the parameters used for inputs of unknown size.
 * `--`:
     All arguments after `--` are treated as files
 
 
-### gzip Operation modifiers
+### gzip Operation Modifiers
 When invoked via a `gzip` symlink, `zstd` will support further
 options that intend to mimic the `gzip` behavior:
 
@@ -300,7 +311,7 @@ options that intend to mimic the `gzip` behavior:
     alias to the option `-9`.
 
 
-### Interactions with Environment Variables
+### Environment Variables
 
 Employing environment variables to set parameters has security implications.
 Therefore, this avenue is intentionally limited.
@@ -341,7 +352,7 @@ Compression of small files similar to the sample set will be greatly improved.
     Since dictionary compression is mostly effective for small files,
     the expectation is that the training set will only contain small files.
     In the case where some samples happen to be large,
-    only the first 128 KB of these samples will be used for training.
+    only the first 128 KiB of these samples will be used for training.
 
     `--train` supports multithreading if `zstd` is compiled with threading support (default).
     Additional advanced parameters can be specified with `--train-fastcover`.
@@ -389,11 +400,13 @@ Compression of small files similar to the sample set will be greatly improved.
     It's possible to provide an explicit number ID instead.
     It's up to the dictionary manager to not assign twice the same ID to
     2 different dictionaries.
-    Note that short numbers have an advantage :
+    Note that short numbers have an advantage:
     an ID < 256 will only need 1 byte in the compressed frame header,
     and an ID < 65536 will only need 2 bytes.
     This compares favorably to 4 bytes default.
 
+    Note that RFC8878 reserves IDs less than 32768 and greater than or equal to 2\^31, so they should not be used in public.
+
 * `--train-cover[=k#,d=#,steps=#,split=#,shrink[=#]]`:
     Select parameters for the default dictionary builder algorithm named cover.
     If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8.
@@ -482,7 +495,7 @@ BENCHMARK
 * `--priority=rt`:
     set process priority to real-time
 
-**Output Format:** CompressionLevel#Filename : InputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
+**Output Format:** CompressionLevel#Filename: InputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
 
 **Methodology:** For both compression and decompression speed, the entire input is compressed/decompressed in-memory to measure speed. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy.
 
@@ -499,9 +512,10 @@ This minimum is either 512 KB, or `overlapSize`, whichever is largest.
 Different job sizes will lead to non-identical compressed frames.
 
 ### --zstd[=options]:
-`zstd` provides 22 predefined compression levels.
-The selected or default predefined compression level can be changed with
-advanced compression options.
+`zstd` provides 22 predefined regular compression levels plus the fast levels.
+This compression level is translated internally into a number of specific parameters that actually control the behavior of the compressor.
+(You can see the result of this translation with `--show-default-cparams`.)
+These specific parameters can be overridden with advanced compression options.
 The _options_ are provided as a comma-separated list.
 You may specify only the options you want to change and the rest will be
 taken from the selected or default compression level.
@@ -510,10 +524,10 @@ The list of available _options_:
 - `strategy`=_strat_, `strat`=_strat_:
     Specify a strategy used by a match finder.
 
-    There are 9 strategies numbered from 1 to 9, from faster to stronger:
-    1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy,
-    4=ZSTD\_lazy, 5=ZSTD\_lazy2, 6=ZSTD\_btlazy2,
-    7=ZSTD\_btopt, 8=ZSTD\_btultra, 9=ZSTD\_btultra2.
+    There are 9 strategies numbered from 1 to 9, from fastest to strongest:
+    1=`ZSTD_fast`, 2=`ZSTD_dfast`, 3=`ZSTD_greedy`,
+    4=`ZSTD_lazy`, 5=`ZSTD_lazy2`, 6=`ZSTD_btlazy2`,
+    7=`ZSTD_btopt`, 8=`ZSTD_btultra`, 9=`ZSTD_btultra2`.
 
 - `windowLog`=_wlog_, `wlog`=_wlog_:
     Specify the maximum number of bits for a match distance.
@@ -533,19 +547,20 @@ The list of available _options_:
     Bigger hash tables cause fewer collisions which usually makes compression
     faster, but requires more memory during compression.
 
-    The minimum _hlog_ is 6 (64 B) and the maximum is 30 (1 GiB).
+    The minimum _hlog_ is 6 (64 entries / 256 B) and the maximum is 30 (1B entries / 4 GiB).
 
 - `chainLog`=_clog_, `clog`=_clog_:
-    Specify the maximum number of bits for a hash chain or a binary tree.
+    Specify the maximum number of bits for the secondary search structure,
+    whose form depends on the selected `strategy`.
 
     Higher numbers of bits increases the chance to find a match which usually
     improves compression ratio.
     It also slows down compression speed and increases memory requirements for
     compression.
-    This option is ignored for the ZSTD_fast strategy.
+    This option is ignored for the `ZSTD_fast` `strategy`, which only has the primary hash table.
 
-    The minimum _clog_ is 6 (64 B) and the maximum is 29 (524 Mib) on 32-bit platforms
-    and 30 (1 Gib) on 64-bit platforms.
+    The minimum _clog_ is 6 (64 entries / 256 B) and the maximum is 29 (512M entries / 2 GiB) on 32-bit platforms
+    and 30 (1B entries / 4 GiB) on 64-bit platforms.
 
 - `searchLog`=_slog_, `slog`=_slog_:
     Specify the maximum number of searches in a hash chain or a binary tree
@@ -567,19 +582,19 @@ The list of available _options_:
 - `targetLength`=_tlen_, `tlen`=_tlen_:
     The impact of this field vary depending on selected strategy.
 
-    For ZSTD\_btopt, ZSTD\_btultra and ZSTD\_btultra2, it specifies
+    For `ZSTD_btopt`, `ZSTD_btultra` and `ZSTD_btultra2`, it specifies
     the minimum match length that causes match finder to stop searching.
     A larger `targetLength` usually improves compression ratio
     but decreases compression speed.
-t
-    For ZSTD\_fast, it triggers ultra-fast mode when > 0.
+
+    For `ZSTD_fast`, it triggers ultra-fast mode when > 0.
     The value represents the amount of data skipped between match sampling.
-    Impact is reversed : a larger `targetLength` increases compression speed
+    Impact is reversed: a larger `targetLength` increases compression speed
     but decreases compression ratio.
 
     For all other strategies, this field has no impact.
 
-    The minimum _tlen_ is 0 and the maximum is 128 Kib.
+    The minimum _tlen_ is 0 and the maximum is 128 KiB.
 
 - `overlapLog`=_ovlog_,  `ovlog`=_ovlog_:
     Determine `overlapSize`, amount of data reloaded from previous job.
@@ -591,7 +606,7 @@ t
     9 means "full overlap", meaning up to `windowSize` is reloaded from previous job.
     Reducing _ovlog_ by 1 reduces the reloaded amount by a factor 2.
     For example, 8 means "windowSize/2", and 6 means "windowSize/8".
-    Value 0 is special and means "default" : _ovlog_ is automatically determined by `zstd`.
+    Value 0 is special and means "default": _ovlog_ is automatically determined by `zstd`.
     In which case, _ovlog_ will range from 6 to 9, depending on selected _strat_.
 
 - `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_:
@@ -641,6 +656,11 @@ similar to predefined level 19 for files bigger than 256 KB:
 
 `--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
 
+SEE ALSO
+--------
+`zstdgrep`(1), `zstdless`(1), `gzip`(1), `xz`(1)
+
+The  format is specified in Y. Collet, "Zstandard Compression and the 'application/zstd' Media Type", https://www.ietf.org/rfc/rfc8878.txt, Internet RFC 8878 (February 2021).
 
 BUGS
 ----
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 362f320a9..93f75e21d 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -27,8 +27,8 @@
 /*-************************************
 *  Dependencies
 **************************************/
-#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
-#include "util.h"     /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
+#include "platform.h" /* PLATFORM_POSIX_VERSION */
+#include "util.h"     /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList, UTIL_isConsole */
 #include    /* getenv */
 #include    /* strcmp, strlen */
 #include     /* fprintf(), stdin, stdout, stderr */
@@ -52,12 +52,12 @@
 /*-************************************
 *  Constants
 **************************************/
-#define COMPRESSOR_NAME "zstd command line interface"
+#define COMPRESSOR_NAME "Zstandard CLI"
 #ifndef ZSTD_VERSION
 #  define ZSTD_VERSION "v" ZSTD_VERSION_STRING
 #endif
 #define AUTHOR "Yann Collet"
-#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
+#define WELCOME_MESSAGE "*** %s (%i-bit) %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
 
 #define ZSTD_ZSTDMT "zstdmt"
 #define ZSTD_UNZSTD "unzstd"
@@ -143,160 +143,174 @@ static int exeNameMatch(const char* exeName, const char* test)
  */
 static void usage(FILE* f, const char* programName)
 {
-    DISPLAY_F(f, "Usage: %s [OPTION]... [FILE]... [-o file]\n", programName);
-    DISPLAY_F(f, "Compress or uncompress FILEs (with no FILE or when FILE is `-`, read from standard input).\n\n");
-    DISPLAY_F(f, "  -o file              result stored into `file` (only 1 output file)\n");
-#ifndef ZSTD_NOCOMPRESS
-    DISPLAY_F(f, "  -1 .. -%d            compression level (faster .. better; default: %d)\n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
-#endif
-#ifndef ZSTD_NODECOMPRESS
-    DISPLAY_F(f, "  -d, --decompress     decompression\n");
-#endif
-    DISPLAY_F(f, "  -f, --force          disable input and output checks. Allows overwriting existing files,\n");
-    DISPLAY_F(f, "                       input from console, output to stdout, operating on links,\n");
-    DISPLAY_F(f, "                       block devices, etc. During decompression and when the output\n");
-    DISPLAY_F(f, "                       destination is stdout, pass-through unrecognized formats as-is.\n");
-    DISPLAY_F(f, "      --rm             remove source file(s) after successful de/compression\n");
-    DISPLAY_F(f, "  -k, --keep           preserve source file(s) (default) \n");
+    DISPLAY_F(f, "Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.\n\n");
+    DISPLAY_F(f, "Usage: %s [OPTIONS...] [INPUT... | -] [-o OUTPUT]\n\n", programName);
+    DISPLAY_F(f, "Options:\n");
+    DISPLAY_F(f, "  -o OUTPUT                     Write output to a single file, OUTPUT.\n");
+    DISPLAY_F(f, "  -k, --keep                    Preserve INPUT file(s). [Default] \n");
+    DISPLAY_F(f, "  --rm                          Remove INPUT file(s) after successful (de)compression.\n");
 #ifdef ZSTD_GZCOMPRESS
     if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
-        DISPLAY_F(f, "  -n, --no-name        do not store original filename when compressing\n");
+        DISPLAY_F(f, "  -n, --no-name                 Do not store original filename when compressing.\n\n");
     }
 #endif
-    DISPLAY_F(f, "  -D DICT              use DICT as Dictionary for compression or decompression\n");
-    DISPLAY_F(f, "  -h                   display usage and exit\n");
-    DISPLAY_F(f, "  -H,--help            display long help and exit\n");
+    DISPLAY_F(f, "\n");
+#ifndef ZSTD_NOCOMPRESS
+    DISPLAY_F(f, "  -#                            Desired compression level, where `#` is a number between 1 and %d;\n", ZSTDCLI_CLEVEL_MAX);
+    DISPLAY_F(f, "                                lower numbers provide faster compression, higher numbers yield\n");
+    DISPLAY_F(f, "                                better compression ratios. [Default: %d]\n\n", ZSTDCLI_CLEVEL_DEFAULT);
+#endif
+#ifndef ZSTD_NODECOMPRESS
+    DISPLAY_F(f, "  -d, --decompress              Perform decompression.\n");
+#endif
+    DISPLAY_F(f, "  -D DICT                       Use DICT as the dictionary for compression or decompression.\n\n");
+    DISPLAY_F(f, "  -f, --force                   Disable input and output checks. Allows overwriting existing files,\n");
+    DISPLAY_F(f, "                                receiving input from the console, printing ouput to STDOUT, and\n");
+    DISPLAY_F(f, "                                operating on links, block devices, etc. Unrecognized formats will be\n");
+    DISPLAY_F(f, "                                passed-through through as-is.\n\n");
+
+    DISPLAY_F(f, "  -h                            Display short usage and exit.\n");
+    DISPLAY_F(f, "  -H, --help                    Display full help and exit.\n");
+    DISPLAY_F(f, "  -V, --version                 Display the program version and exit.\n");
     DISPLAY_F(f, "\n");
 }
 
 static void usage_advanced(const char* programName)
 {
     DISPLAYOUT(WELCOME_MESSAGE);
+    DISPLAYOUT("\n");
     usage(stdout, programName);
-    DISPLAYOUT("Advanced options :\n");
-    DISPLAYOUT("  -V, --version        display Version number and exit\n");
+    DISPLAYOUT("Advanced options:\n");
+    DISPLAYOUT("  -c, --stdout                  Write to STDOUT (even if it is a console) and keep the INPUT file(s).\n\n");
 
-    DISPLAYOUT("  -c, --stdout         write to standard output (even if it is the console), keep original file\n");
-
-    DISPLAYOUT("  -v, --verbose        verbose mode; specify multiple times to increase verbosity\n");
-    DISPLAYOUT("  -q, --quiet          suppress warnings; specify twice to suppress errors too\n");
-    DISPLAYOUT("      --[no-]progress  forcibly display, or never display the progress counter\n");
-    DISPLAYOUT("                       note: any (de)compressed output to terminal will mix with progress counter text\n");
+    DISPLAYOUT("  -v, --verbose                 Enable verbose output; pass multiple times to increase verbosity.\n");
+    DISPLAYOUT("  -q, --quiet                   Suppress warnings; pass twice to suppress errors.\n");
+#ifndef ZSTD_NOTRACE
+    DISPLAYOUT("  --trace LOG                   Log tracing information to LOG.\n");
+#endif
+    DISPLAYOUT("\n");
+    DISPLAYOUT("  --[no-]progress               Forcibly show/hide the progress counter. NOTE: Any (de)compressed\n");
+    DISPLAYOUT("                                output to terminal will mix with progress counter text.\n\n");
 
 #ifdef UTIL_HAS_CREATEFILELIST
-    DISPLAYOUT("  -r                   operate recursively on directories\n");
-    DISPLAYOUT("     --filelist FILE   read list of files to operate upon from FILE\n");
-    DISPLAYOUT("     --output-dir-flat DIR :  processed files are stored into DIR\n");
+    DISPLAYOUT("  -r                            Operate recursively on directories.\n");
+    DISPLAYOUT("  --filelist LIST               Read a list of files to operate on from LIST.\n");
+    DISPLAYOUT("  --output-dir-flat DIR         Store processed files in DIR.\n");
 #endif
 
 #ifdef UTIL_HAS_MIRRORFILELIST
-    DISPLAYOUT("     --output-dir-mirror DIR :  processed files are stored into DIR respecting original directory structure\n");
+    DISPLAYOUT("  --output-dir-mirror DIR       Store processed files in DIR, respecting original directory structure.\n");
 #endif
     if (AIO_supported())
-        DISPLAYOUT("     --[no-]asyncio    use asynchronous IO (default: enabled)\n");
+        DISPLAYOUT("  --[no-]asyncio                Use asynchronous IO. [Default: Enabled]\n");
 
+    DISPLAYOUT("\n");
 #ifndef ZSTD_NOCOMPRESS
-    DISPLAYOUT("     --[no-]check      during compression, add XXH64 integrity checksum to frame (default: enabled)\n");
+    DISPLAYOUT("  --[no-]check                  Add XXH64 integrity checksums during compression. [Default: Add, Validate]\n");
 #ifndef ZSTD_NODECOMPRESS
-    DISPLAYOUT("                       if specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate)\n");
+    DISPLAYOUT("                                If `-d` is present, ignore/validate checksums during decompression.\n");
 #endif
 #else
 #ifdef ZSTD_NOCOMPRESS
-    DISPLAYOUT("      --[no-]check     during decompression, ignore/validate checksums in compressed frame (default: validate)");
+    DISPLAYOUT("  --[no-]check                  Ignore/validate checksums during decompression. [Default: Validate]");
 #endif
-    DISPLAYOUT("\n");
 #endif /* ZSTD_NOCOMPRESS */
 
-#ifndef ZSTD_NOTRACE
-    DISPLAYOUT("     --trace FILE      log tracing information to FILE\n");
-#endif
-    DISPLAYOUT("     --                all arguments after \"--\" are treated as files\n");
+    DISPLAYOUT("\n");
+    DISPLAYOUT("  --                            Treat remaining arguments after `--` as files.\n");
 
 #ifndef ZSTD_NOCOMPRESS
     DISPLAYOUT("\n");
-    DISPLAYOUT("Advanced compression options :\n");
-    DISPLAYOUT("     --ultra           enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
-    DISPLAYOUT("     --fast[=#]        switch to very fast compression levels (default: %u)\n", 1);
+    DISPLAYOUT("Advanced compression options:\n");
+    DISPLAYOUT("  --ultra                       Enable levels beyond %i, up to %i; requires more memory.\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
+    DISPLAYOUT("  --fast[=#]                    Use to very fast compression levels. [Default: %u]\n", 1);
 #ifdef ZSTD_GZCOMPRESS
     if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
-        DISPLAYOUT("     --best            compatibility alias for -9 \n");
-        DISPLAYOUT("     --no-name         do not store original filename when compressing\n");
+        DISPLAYOUT("  --best                        Compatibility alias for `-9`.\n");
     }
 #endif
-    DISPLAYOUT("     --long[=#]        enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
-    DISPLAYOUT("     --patch-from=FILE :  specify the file to be used as a reference point for zstd's diff engine. \n");
-    DISPLAYOUT("     --adapt           dynamically adapt compression level to I/O conditions\n");
+    DISPLAYOUT("  --adapt                       Dynamically adapt compression level to I/O conditions.\n");
+    DISPLAYOUT("  --long[=#]                    Enable long distance matching with window log #. [Default: %u]\n", g_defaultMaxWindowLog);
+    DISPLAYOUT("  --patch-from=REF              Use REF as the reference point for Zstandard's diff engine. \n\n");
 # ifdef ZSTD_MULTITHREAD
-    DISPLAYOUT("  -T#                  spawn # compression threads (default: 1, 0==# cores) \n");
-    DISPLAYOUT("  -B#                  select size of each job (default: 0==automatic) \n");
-    DISPLAYOUT("     --single-thread   use a single thread for both I/O and compression (result slightly different than -T1) \n");
-    DISPLAYOUT("     --auto-threads={physical,logical} :  use either physical cores or logical cores as default when specifying -T0 (default: physical)\n");
-    DISPLAYOUT("     --rsyncable       compress using a rsync-friendly method (-B sets block size) \n");
+    DISPLAYOUT("  -T#                           Spawn # compression threads. [Default: 1; pass 0 for core count.]\n");
+    DISPLAYOUT("  --single-thread               Share a single thread for I/O and compression (slightly different than `-T1`).\n");
+    DISPLAYOUT("  --auto-threads={physical|logical}\n");
+    DISPLAYOUT("                                Use physical/logical cores when using `-T0`. [Default: Physical]\n\n");
+    DISPLAYOUT("  -B#                           Set job size to #. [Default: 0 (automatic)]\n");
+    DISPLAYOUT("  --rsyncable                   Compress using a rsync-friendly method (`-B` sets block size). \n");
+    DISPLAYOUT("\n");
 # endif
-    DISPLAYOUT("     --exclude-compressed :  only compress files that are not already compressed \n");
-    DISPLAYOUT("     --stream-size=#   specify size of streaming input from `stdin` \n");
-    DISPLAYOUT("     --size-hint=#     optimize compression parameters for streaming input of approximately this size \n");
-    DISPLAYOUT("     --target-compressed-block-size=# :  generate compressed block of approximately targeted size \n");
-    DISPLAYOUT("     --no-dictID       don't write dictID into header (dictionary compression only)\n");
-    DISPLAYOUT("     --[no-]compress-literals :  force (un)compressed literals\n");
-    DISPLAYOUT("     --[no-]row-match-finder :  force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
+    DISPLAYOUT("  --exclude-compressed          Only compress files that are not already compressed.\n\n");
 
-    DISPLAYOUT("     --format=zstd     compress files to the .zst format (default)\n");
+    DISPLAYOUT("  --stream-size=#               Specify size of streaming input from STDIN.\n");
+    DISPLAYOUT("  --size-hint=#                 Optimize compression parameters for streaming input of approximately size #.\n");
+    DISPLAYOUT("  --target-compressed-block-size=#\n");
+    DISPLAYOUT("                                Generate compressed blocks of approximately # size.\n\n");
+    DISPLAYOUT("  --no-dictID                   Don't write `dictID` into the header (dictionary compression only).\n");
+    DISPLAYOUT("  --[no-]compress-literals      Force (un)compressed literals.\n");
+    DISPLAYOUT("  --[no-]row-match-finder       Explicitly enable/disable the fast, row-based matchfinder for\n");
+    DISPLAYOUT("                                the 'greedy', 'lazy', and 'lazy2' strategies.\n");
+
+    DISPLAYOUT("\n");
+    DISPLAYOUT("  --format=zstd                 Compress files to the `.zst` format. [Default]\n");
 #ifdef ZSTD_GZCOMPRESS
-    DISPLAYOUT("     --format=gzip     compress files to the .gz format\n");
+    DISPLAYOUT("  --format=gzip                 Compress files to the `.gz` format.\n");
 #endif
 #ifdef ZSTD_LZMACOMPRESS
-    DISPLAYOUT("     --format=xz       compress files to the .xz format\n");
-    DISPLAYOUT("     --format=lzma     compress files to the .lzma format\n");
+    DISPLAYOUT("  --format=xz                   Compress files to the `.xz` format.\n");
+    DISPLAYOUT("  --format=lzma                 Compress files to the `.lzma` format.\n");
 #endif
 #ifdef ZSTD_LZ4COMPRESS
-    DISPLAYOUT( "     --format=lz4     compress files to the .lz4 format\n");
+    DISPLAYOUT( "  --format=lz4                 Compress files to the `.lz4` format.\n");
 #endif
 #endif  /* !ZSTD_NOCOMPRESS */
 
 #ifndef ZSTD_NODECOMPRESS
     DISPLAYOUT("\n");
-    DISPLAYOUT("Advanced decompression options :\n");
-    DISPLAYOUT("  -l                   print information about zstd compressed files\n");
-    DISPLAYOUT("     --test            test compressed file integrity\n");
-    DISPLAYOUT("  -M#                  Set a memory usage limit for decompression\n");
+    DISPLAYOUT("Advanced decompression options:\n");
+    DISPLAYOUT("  -l                            Print information about Zstandard-compressed files.\n");
+    DISPLAYOUT("  --test                        Test compressed file integrity.\n");
+    DISPLAYOUT("  -M#                           Set the memory usage limit to # megabytes.\n");
 # if ZSTD_SPARSE_DEFAULT
-    DISPLAYOUT("     --[no-]sparse     sparse mode (default: enabled on file, disabled on stdout)\n");
+    DISPLAYOUT("  --[no-]sparse                 Enable sparse mode. [Default: Enabled for files, disabled for STDOUT.]\n");
 # else
-    DISPLAYOUT("     --[no-]sparse     sparse mode (default: disabled)\n");
+    DISPLAYOUT("  --[no-]sparse                 Enable sparse mode. [Default: Disabled]\n");
 # endif
     {
-        char const* passThroughDefault = "disabled";
+        char const* passThroughDefault = "Disabled";
         if (exeNameMatch(programName, ZSTD_CAT) ||
             exeNameMatch(programName, ZSTD_ZCAT) ||
             exeNameMatch(programName, ZSTD_GZCAT)) {
-            passThroughDefault = "enabled";
+            passThroughDefault = "Enabled";
         }
-        DISPLAYOUT("     --[no-]pass-through : passes through uncompressed files as-is (default: %s)\n", passThroughDefault);
+        DISPLAYOUT("  --[no-]pass-through           Pass through uncompressed files as-is. [Default: %s]\n", passThroughDefault);
     }
 #endif  /* ZSTD_NODECOMPRESS */
 
 #ifndef ZSTD_NODICT
     DISPLAYOUT("\n");
-    DISPLAYOUT("Dictionary builder :\n");
-    DISPLAYOUT("     --train ##        create a dictionary from a training set of files\n");
-    DISPLAYOUT("     --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] :  use the cover algorithm with optional args\n");
-    DISPLAYOUT("     --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] :  use the fast cover algorithm with optional args\n");
-    DISPLAYOUT("     --train-legacy[=s=#] :  use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
-    DISPLAYOUT("  -o DICT              DICT is dictionary name (default: %s)\n", g_defaultDictName);
-    DISPLAYOUT("     --maxdict=#       limit dictionary to specified size (default: %u)\n", g_defaultMaxDictSize);
-    DISPLAYOUT("     --dictID=#        force dictionary ID to specified value (default: random)\n");
+    DISPLAYOUT("Dictionary builder:\n");
+    DISPLAYOUT("  --train                       Create a dictionary from a training set of files.\n\n");
+    DISPLAYOUT("  --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]]\n");
+    DISPLAYOUT("                                Use the cover algorithm (with optional arguments).\n");
+    DISPLAYOUT("  --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]]\n");
+    DISPLAYOUT("                                Use the fast cover algorithm (with optional arguments).\n\n");
+    DISPLAYOUT("  --train-legacy[=s=#]          Use the legacy algorithm with selectivity #. [Default: %u]\n", g_defaultSelectivityLevel);
+    DISPLAYOUT("  -o NAME                       Use NAME as dictionary name. [Default: %s]\n", g_defaultDictName);
+    DISPLAYOUT("  --maxdict=#                   Limit dictionary to specified size #. [Default: %u]\n", g_defaultMaxDictSize);
+    DISPLAYOUT("  --dictID=#                    Force dictionary ID to #. [Default: Random]\n");
 #endif
 
 #ifndef ZSTD_NOBENCH
     DISPLAYOUT("\n");
-    DISPLAYOUT("Benchmark options : \n");
-    DISPLAYOUT("  -b#                  benchmark file(s), using # compression level (default: %d)\n", ZSTDCLI_CLEVEL_DEFAULT);
-    DISPLAYOUT("  -e#                  test all compression levels successively from -b# to -e# (default: 1)\n");
-    DISPLAYOUT("  -i#                  minimum evaluation time in seconds (default: 3s)\n");
-    DISPLAYOUT("  -B#                  cut file into independent chunks of size # (default: no chunking)\n");
-    DISPLAYOUT("  -S                   output one benchmark result per input file (default: consolidated result)\n");
-    DISPLAYOUT("     --priority=rt     set process priority to real-time\n");
+    DISPLAYOUT("Benchmark options:\n");
+    DISPLAYOUT("  -b#                           Perform benchmarking with compression level #. [Default: %d]\n", ZSTDCLI_CLEVEL_DEFAULT);
+    DISPLAYOUT("  -e#                           Test all compression levels up to #; starting level is `-b#`. [Default: 1]\n");
+    DISPLAYOUT("  -i#                           Set the minimum evaluation to time # seconds. [Default: 3]\n");
+    DISPLAYOUT("  -B#                           Cut file into independent chunks of size #. [Default: No chunking]\n");
+    DISPLAYOUT("  -S                            Output one benchmark result per input file. [Default: Consolidated result]\n");
+    DISPLAYOUT("  --priority=rt                 Set process priority to real-time.\n");
 #endif
 
 }
@@ -325,7 +339,7 @@ static const char* lastNameFromPath(const char* path)
 
 static void errorOut(const char* msg)
 {
-    DISPLAY("%s \n", msg); exit(1);
+    DISPLAYLEVEL(1, "%s \n", msg); exit(1);
 }
 
 /*! readU32FromCharChecked() :
@@ -772,13 +786,13 @@ static unsigned init_nbThreads(void) {
     } else {                      \
         argNb++;                  \
         if (argNb >= argCount) {  \
-            DISPLAY("error: missing command argument \n"); \
+            DISPLAYLEVEL(1, "error: missing command argument \n"); \
             CLEAN_RETURN(1);      \
         }                         \
         ptr = argv[argNb];        \
         assert(ptr != NULL);      \
         if (ptr[0]=='-') {        \
-            DISPLAY("error: command cannot be separated from its argument by another command \n"); \
+            DISPLAYLEVEL(1, "error: command cannot be separated from its argument by another command \n"); \
             CLEAN_RETURN(1);      \
 }   }   }
 
@@ -824,7 +838,6 @@ int main(int argCount, const char* argv[])
         ldmFlag = 0,
         main_pause = 0,
         adapt = 0,
-        useRowMatchFinder = 0,
         adaptMin = MINCLEVEL,
         adaptMax = MAXCLEVEL,
         rsyncable = 0,
@@ -836,7 +849,10 @@ int main(int argCount, const char* argv[])
         defaultLogicalCores = 0,
         showDefaultCParams = 0,
         ultra=0,
-        contentSize=1;
+        contentSize=1,
+        removeSrcFile=0;
+    ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
+    FIO_compressionType_t cType = FIO_zstdCompression;
     unsigned nbWorkers = 0;
     double compressibility = 0.5;
     unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
@@ -844,6 +860,7 @@ int main(int argCount, const char* argv[])
 
     FIO_prefs_t* const prefs = FIO_createPreferences();
     FIO_ctx_t* const fCtx = FIO_createContext();
+    FIO_progressSetting_e progress = FIO_ps_auto;
     zstd_operation_mode operation = zom_compress;
     ZSTD_compressionParameters compressionParams;
     int cLevel = init_cLevel();
@@ -883,7 +900,7 @@ int main(int argCount, const char* argv[])
     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
     (void)memLimit;
     assert(argCount >= 1);
-    if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAY("zstd: allocation error \n"); exit(1); }
+    if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAYLEVEL(1, "zstd: allocation error \n"); exit(1); }
     programName = lastNameFromPath(programName);
 #ifdef ZSTD_MULTITHREAD
     nbWorkers = init_nbThreads();
@@ -895,17 +912,17 @@ int main(int argCount, const char* argv[])
     if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }     /* supports multiple formats */
     if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }    /* behave like zcat, also supports multiple formats */
     if (exeNameMatch(programName, ZSTD_GZ)) {   /* behave like gzip */
-        suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1);
+        suffix = GZ_EXTENSION; cType = FIO_gzipCompression; removeSrcFile=1;
         dictCLevel = cLevel = 6;  /* gzip default is -6 */
     }
-    if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); }                                                     /* behave like gunzip, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; removeSrcFile=1; }                                                     /* behave like gunzip, also supports multiple formats */
     if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }   /* behave like gzcat, also supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); }    /* behave like lzma */
-    if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); }          /* behave like xz */
-    if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); }     /* behave like unxz, also supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); }                                       /* behave like lz4 */
-    if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lz4Compression); }                                   /* behave like unlz4, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; cType = FIO_lzmaCompression; removeSrcFile=1; }    /* behave like lzma */
+    if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; cType = FIO_lzmaCompression; removeSrcFile=1; } /* behave like unlzma, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; cType = FIO_xzCompression; removeSrcFile=1; }          /* behave like xz */
+    if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; cType = FIO_xzCompression; removeSrcFile=1; }     /* behave like unxz, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; cType = FIO_lz4Compression; }                        /* behave like lz4 */
+    if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; cType = FIO_lz4Compression; }                    /* behave like unlz4, also supports multiple formats */
     memset(&compressionParams, 0, sizeof(compressionParams));
 
     /* init crash handler */
@@ -942,7 +959,7 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--help")) { usage_advanced(programName); CLEAN_RETURN(0); }
                 if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
                 if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
-                if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; FIO_setRemoveSrcFile(prefs, 0); g_displayLevel-=(g_displayLevel==2); continue; }
+                if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; removeSrcFile=0; continue; }
                 if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
                 if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(prefs, 2); continue; }
                 if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
@@ -955,38 +972,42 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--no-asyncio")) { FIO_setAsyncIOFlag(prefs, 0); continue;}
                 if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; }
                 if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(prefs, 0); continue; }
-                if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; }
-                if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
+                if (!strcmp(argument, "--keep")) { removeSrcFile=0; continue; }
+                if (!strcmp(argument, "--rm")) { removeSrcFile=1; continue; }
                 if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
                 if (!strcmp(argument, "--show-default-cparams")) { showDefaultCParams = 1; continue; }
                 if (!strcmp(argument, "--content-size")) { contentSize = 1; continue; }
                 if (!strcmp(argument, "--no-content-size")) { contentSize = 0; continue; }
                 if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
-                if (!strcmp(argument, "--no-row-match-finder")) { useRowMatchFinder = 1; continue; }
-                if (!strcmp(argument, "--row-match-finder")) { useRowMatchFinder = 2; continue; }
+                if (!strcmp(argument, "--no-row-match-finder")) { useRowMatchFinder = ZSTD_ps_disable; continue; }
+                if (!strcmp(argument, "--row-match-finder")) { useRowMatchFinder = ZSTD_ps_enable; continue; }
                 if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; }
                 if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
-                if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(prefs, FIO_zstdCompression); continue; }
+                if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; cType = FIO_zstdCompression; continue; }
 #ifdef ZSTD_GZCOMPRESS
-                if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); continue; }
+                if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; cType = FIO_gzipCompression; continue; }
                 if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
                     if (!strcmp(argument, "--best")) { dictCLevel = cLevel = 9; continue; }
                     if (!strcmp(argument, "--no-name")) { /* ignore for now */; continue; }
                 }
 #endif
 #ifdef ZSTD_LZMACOMPRESS
-                if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression);  continue; }
-                if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression);  continue; }
+                if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; cType = FIO_lzmaCompression; continue; }
+                if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; cType = FIO_xzCompression; continue; }
 #endif
 #ifdef ZSTD_LZ4COMPRESS
-                if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression);  continue; }
+                if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; cType = FIO_lz4Compression; continue; }
 #endif
                 if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
                 if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_ps_enable; continue; }
                 if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_ps_disable; continue; }
-                if (!strcmp(argument, "--no-progress")) { FIO_setProgressSetting(FIO_ps_never); continue; }
-                if (!strcmp(argument, "--progress")) { FIO_setProgressSetting(FIO_ps_always); continue; }
+                if (!strcmp(argument, "--no-progress")) { progress = FIO_ps_never; continue; }
+                if (!strcmp(argument, "--progress")) { progress = FIO_ps_always; continue; }
                 if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
+                if (!strcmp(argument, "--fake-stdin-is-console")) { UTIL_fakeStdinIsConsole(); continue; }
+                if (!strcmp(argument, "--fake-stdout-is-console")) { UTIL_fakeStdoutIsConsole(); continue; }
+                if (!strcmp(argument, "--fake-stderr-is-console")) { UTIL_fakeStderrIsConsole(); continue; }
+                if (!strcmp(argument, "--trace-file-stat")) { UTIL_traceFileStat(); continue; }
 
                 /* long commands with arguments */
 #ifndef ZSTD_NODICT
@@ -1031,14 +1052,14 @@ int main(int argCount, const char* argv[])
                 if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(blockSize); continue; }
                 if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; }
                 if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
-                if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } continue; }
+                if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; }
                 if (longCommandWArg(&argument, "--stream-size")) { NEXT_TSIZE(streamSrcSize); continue; }
                 if (longCommandWArg(&argument, "--target-compressed-block-size")) { NEXT_TSIZE(targetCBlockSize); continue; }
                 if (longCommandWArg(&argument, "--size-hint")) { NEXT_TSIZE(srcSizeHint); continue; }
                 if (longCommandWArg(&argument, "--output-dir-flat")) {
                     NEXT_FIELD(outDirName);
                     if (strlen(outDirName) == 0) {
-                        DISPLAY("error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
+                        DISPLAYLEVEL(1, "error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
                         CLEAN_RETURN(1);
                     }
                     continue;
@@ -1054,7 +1075,7 @@ int main(int argCount, const char* argv[])
                 if (longCommandWArg(&argument, "--output-dir-mirror")) {
                     NEXT_FIELD(outMirroredDirName);
                     if (strlen(outMirroredDirName) == 0) {
-                        DISPLAY("error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
+                        DISPLAYLEVEL(1, "error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
                         CLEAN_RETURN(1);
                     }
                     continue;
@@ -1149,7 +1170,7 @@ int main(int argCount, const char* argv[])
                         operation=zom_decompress; argument++; break;
 
                     /* Force stdout, even if stdout==console */
-                case 'c': forceStdout=1; outFileName=stdoutmark; FIO_setRemoveSrcFile(prefs, 0); argument++; break;
+                case 'c': forceStdout=1; outFileName=stdoutmark; removeSrcFile=0; argument++; break;
 
                     /* do not store filename - gzip compatibility - nothing to do */
                 case 'n': argument++; break;
@@ -1167,7 +1188,7 @@ int main(int argCount, const char* argv[])
                 case 'q': g_displayLevel--; argument++; break;
 
                     /* keep source file (default) */
-                case 'k': FIO_setRemoveSrcFile(prefs, 0); argument++; break;
+                case 'k': removeSrcFile=0; argument++; break;
 
                     /* Checksum */
                 case 'C': FIO_setChecksumFlag(prefs, 2); argument++; break;
@@ -1330,7 +1351,7 @@ int main(int argCount, const char* argv[])
         int const ret = FIO_listMultipleFiles((unsigned)filenames->tableSize, filenames->fileNames, g_displayLevel);
         CLEAN_RETURN(ret);
 #else
-        DISPLAY("file information is not supported \n");
+        DISPLAYLEVEL(1, "file information is not supported \n");
         CLEAN_RETURN(1);
 #endif
     }
@@ -1338,6 +1359,10 @@ int main(int argCount, const char* argv[])
     /* Check if benchmark is selected */
     if (operation==zom_bench) {
 #ifndef ZSTD_NOBENCH
+        if (cType != FIO_zstdCompression) {
+            DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n");
+            CLEAN_RETURN(1);
+        }
         benchParams.blockSize = blockSize;
         benchParams.nbWorkers = (int)nbWorkers;
         benchParams.realTime = (unsigned)setRealTimePrio;
@@ -1345,7 +1370,7 @@ int main(int argCount, const char* argv[])
         benchParams.ldmFlag = ldmFlag;
         benchParams.ldmMinMatch = (int)g_ldmMinMatch;
         benchParams.ldmHashLog = (int)g_ldmHashLog;
-        benchParams.useRowMatchFinder = useRowMatchFinder;
+        benchParams.useRowMatchFinder = (int)useRowMatchFinder;
         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
             benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
         }
@@ -1366,15 +1391,18 @@ int main(int argCount, const char* argv[])
                     int c;
                     DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
                     for(c = cLevel; c <= cLevelLast; c++) {
-                        BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
+                        BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
+                        if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
                 }   }
             } else {
                 for(; cLevel <= cLevelLast; cLevel++) {
-                    BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
+                    BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
+                    if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
             }   }
         } else {
             for(; cLevel <= cLevelLast; cLevel++) {
-                BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
+                BMK_benchOutcome_t const bo = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
+                if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
         }   }
 
 #else
@@ -1416,7 +1444,7 @@ int main(int argCount, const char* argv[])
     }
 
 #ifndef ZSTD_NODECOMPRESS
-    if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); }  /* test mode */
+    if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; removeSrcFile=0; }  /* test mode */
 #endif
 
     /* No input filename ==> use stdin and stdout */
@@ -1437,12 +1465,12 @@ int main(int argCount, const char* argv[])
     /* Check if input/output defined as console; trigger an error in this case */
     if (!forceStdin
      && (UTIL_searchFileNamesTable(filenames, stdinmark) != -1)
-     && IS_CONSOLE(stdin) ) {
+     && UTIL_isConsole(stdin) ) {
         DISPLAYLEVEL(1, "stdin is a console, aborting\n");
         CLEAN_RETURN(1);
     }
     if ( (!outFileName || !strcmp(outFileName, stdoutmark))
-      && IS_CONSOLE(stdout)
+      && UTIL_isConsole(stdout)
       && (UTIL_searchFileNamesTable(filenames, stdinmark) != -1)
       && !forceStdout
       && operation!=zom_decompress ) {
@@ -1461,25 +1489,35 @@ int main(int argCount, const char* argv[])
 
     if (showDefaultCParams) {
         if (operation == zom_decompress) {
-            DISPLAY("error : can't use --show-default-cparams in decompression mode \n");
+            DISPLAYLEVEL(1, "error : can't use --show-default-cparams in decompression mode \n");
             CLEAN_RETURN(1);
         }
     }
 
     if (dictFileName != NULL && patchFromDictFileName != NULL) {
-        DISPLAY("error : can't use -D and --patch-from=# at the same time \n");
+        DISPLAYLEVEL(1, "error : can't use -D and --patch-from=# at the same time \n");
         CLEAN_RETURN(1);
     }
 
     if (patchFromDictFileName != NULL && filenames->tableSize > 1) {
-        DISPLAY("error : can't use --patch-from=# on multiple files \n");
+        DISPLAYLEVEL(1, "error : can't use --patch-from=# on multiple files \n");
         CLEAN_RETURN(1);
     }
 
-    /* No status message in pipe mode (stdin - stdout) */
+    /* No status message by default when output is stdout */
     hasStdout = outFileName && !strcmp(outFileName,stdoutmark);
+    if (hasStdout && (g_displayLevel==2)) g_displayLevel=1;
 
-    if ((hasStdout || !IS_CONSOLE(stderr)) && (g_displayLevel==2)) g_displayLevel=1;
+    /* when stderr is not the console, do not pollute it with progress updates (unless requested) */
+    if (!UTIL_isConsole(stderr) && (progress!=FIO_ps_always)) progress=FIO_ps_never;
+    FIO_setProgressSetting(progress);
+
+    /* don't remove source files when output is stdout */;
+    if (hasStdout && removeSrcFile) {
+        DISPLAYLEVEL(3, "Note: src files are not removed when output is stdout \n");
+        removeSrcFile = 0;
+    }
+    FIO_setRemoveSrcFile(prefs, removeSrcFile);
 
     /* IO Stream/File */
     FIO_setHasStdoutOutput(fCtx, hasStdout);
@@ -1499,6 +1537,7 @@ int main(int argCount, const char* argv[])
     FIO_setMemLimit(prefs, memLimit);
     if (operation==zom_compress) {
 #ifndef ZSTD_NOCOMPRESS
+        FIO_setCompressionType(prefs, cType);
         FIO_setContentSize(prefs, contentSize);
         FIO_setNbWorkers(prefs, (int)nbWorkers);
         FIO_setBlockSize(prefs, (int)blockSize);
@@ -1509,7 +1548,7 @@ int main(int argCount, const char* argv[])
         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
         if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
         FIO_setAdaptiveMode(prefs, adapt);
-        FIO_setUseRowMatchFinder(prefs, useRowMatchFinder);
+        FIO_setUseRowMatchFinder(prefs, (int)useRowMatchFinder);
         FIO_setAdaptMin(prefs, adaptMin);
         FIO_setAdaptMax(prefs, adaptMax);
         FIO_setRsyncable(prefs, rsyncable);
@@ -1543,8 +1582,12 @@ int main(int argCount, const char* argv[])
         else
             operationResult = FIO_compressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
 #else
-        (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; (void)useRowMatchFinder; /* not used when ZSTD_NOCOMPRESS set */
-        DISPLAY("Compression not supported \n");
+        /* these variables are only used when compression mode is enabled */
+        (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable;
+        (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode;
+        (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint;
+        (void)ZSTD_strategyMap; (void)useRowMatchFinder; (void)cType;
+        DISPLAYLEVEL(1, "Compression not supported \n");
 #endif
     } else {  /* decompression or test */
 #ifndef ZSTD_NODECOMPRESS
@@ -1554,7 +1597,7 @@ int main(int argCount, const char* argv[])
             operationResult = FIO_decompressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName);
         }
 #else
-        DISPLAY("Decompression not supported \n");
+        DISPLAYLEVEL(1, "Decompression not supported \n");
 #endif
     }
 
diff --git a/programs/zstdcli_trace.c b/programs/zstdcli_trace.c
index b3b977feb..35075a52c 100644
--- a/programs/zstdcli_trace.c
+++ b/programs/zstdcli_trace.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/zstdcli_trace.h b/programs/zstdcli_trace.h
index 38c27dc04..9c135d3ca 100644
--- a/programs/zstdcli_trace.h
+++ b/programs/zstdcli_trace.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/zstdgrep.1.md b/programs/zstdgrep.1.md
index 35186a4bf..6370a81c7 100644
--- a/programs/zstdgrep.1.md
+++ b/programs/zstdgrep.1.md
@@ -4,16 +4,16 @@ zstdgrep(1) -- print lines matching a pattern in zstandard-compressed files
 SYNOPSIS
 --------
 
-`zstdgrep` [*grep-flags*] [--] _pattern_ [_files_ ...]
+`zstdgrep` [] [--]  [ ...]
 
 
 DESCRIPTION
 -----------
-`zstdgrep` runs `grep (1)` on files, or `stdin` if no files argument is given, after decompressing them with `zstdcat (1)`.
+`zstdgrep` runs `grep`(1) on files, or `stdin` if no files argument is given, after decompressing them with `zstdcat`(1).
 
-The grep-flags and pattern arguments are passed on to `grep (1)`.  If an `-e` flag is found in the `grep-flags`, `zstdgrep` will not look for a pattern argument.
+The  and  arguments are passed on to `grep`(1).  If an `-e` flag is found in the , `zstdgrep` will not look for a  argument.
 
-Note that modern `grep` alternatives such as `ripgrep` (`rg`) support `zstd`-compressed files out of the box,
+Note that modern `grep` alternatives such as `ripgrep` (`rg`(1)) support `zstd`-compressed files out of the box,
 and can prove better alternatives than `zstdgrep` notably for unsupported complex pattern searches.
 Note though that such alternatives may also feature some minor command line differences.
 
@@ -23,7 +23,7 @@ In case of missing arguments or missing pattern, 1 will be returned, otherwise 0
 
 SEE ALSO
 --------
-`zstd (1)`
+`zstd`(1)
 
 AUTHORS
 -------
diff --git a/programs/zstdless.1.md b/programs/zstdless.1.md
index d91d48abc..67c1c7676 100644
--- a/programs/zstdless.1.md
+++ b/programs/zstdless.1.md
@@ -4,13 +4,13 @@ zstdless(1) -- view zstandard-compressed files
 SYNOPSIS
 --------
 
-`zstdless` [*flags*] [_file_ ...]
+`zstdless` [] [ ...]
 
 
 DESCRIPTION
 -----------
-`zstdless` runs `less (1)` on files or stdin, if no files argument is given, after decompressing them with `zstdcat (1)`.
+`zstdless` runs `less`(1) on files or stdin, if no  argument is given, after decompressing them with `zstdcat`(1).
 
 SEE ALSO
 --------
-`zstd (1)`
+`zstd`(1)
diff --git a/tests/DEPRECATED-test-zstd-speed.py b/tests/DEPRECATED-test-zstd-speed.py
index 665e0a7f9..71d75b8cc 100755
--- a/tests/DEPRECATED-test-zstd-speed.py
+++ b/tests/DEPRECATED-test-zstd-speed.py
@@ -2,7 +2,7 @@
 # THIS BENCHMARK IS BEING REPLACED BY automated-bencmarking.py
 
 # ################################################################
-# Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/Makefile b/tests/Makefile
index afea6475a..3eed19ea8 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,6 +1,6 @@
 
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -169,7 +169,7 @@ fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PR
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
 
 CLEAN += zstreamtest zstreamtest32
-ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c
+ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c external_matchfinder.c
 ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
 ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
 zstreamtest32 : CFLAGS += -m32
@@ -373,7 +373,7 @@ test-zstream: zstreamtest
 	$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
 
 test-zstream32: zstreamtest32
-	$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+	$(QEMU_SYS) ./zstreamtest32 -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
 
 test-longmatch: longmatch
 	$(QEMU_SYS) ./longmatch
diff --git a/tests/automated_benchmarking.py b/tests/automated_benchmarking.py
index c98391193..153e7db4c 100644
--- a/tests/automated_benchmarking.py
+++ b/tests/automated_benchmarking.py
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/bigdict.c b/tests/bigdict.c
index fb0892586..ff2bb2d70 100644
--- a/tests/bigdict.c
+++ b/tests/bigdict.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/checkTag.c b/tests/checkTag.c
index f6c5e9766..26871ed0f 100644
--- a/tests/checkTag.c
+++ b/tests/checkTag.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/check_size.py b/tests/check_size.py
new file mode 100755
index 000000000..028b0a9c0
--- /dev/null
+++ b/tests/check_size.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+import os
+import subprocess
+import sys
+
+if len(sys.argv) != 3:
+	print(f"Usage: {sys.argv[0]} FILE SIZE_LIMIT")
+	sys.exit(1)
+
+file = sys.argv[1]
+limit = int(sys.argv[2])
+
+if not os.path.exists(file):
+	print(f"{file} does not exist")
+	sys.exit(1)
+
+size = os.path.getsize(file)
+
+if size > limit:
+	print(f"file {file} is {size} bytes, which is greater than the limit of {limit} bytes")
+	sys.exit(1)
diff --git a/tests/cli-tests/README.md b/tests/cli-tests/README.md
index cdf9b8e71..7ca07c3e9 100644
--- a/tests/cli-tests/README.md
+++ b/tests/cli-tests/README.md
@@ -45,6 +45,16 @@ Examples:
 ./run.py --preserve --verbose basic/help.sh
 ```
 
+### Updating exact output
+
+If a test is failing because a `.stderr.exact` or `.stdout.exact` no longer matches, you can re-run the tests with `--set-exact-output` and the correct output will be written.
+
+Example:
+```
+./run.py --set-exact-output
+./run.py basic/help.sh --set-exact-output
+```
+
 ## Writing a test
 
 Test cases are arbitrary executables, and can be written in any language, but are generally shell scripts.
diff --git a/tests/cli-tests/basic/help.sh.stdout.glob b/tests/cli-tests/basic/help.sh.stdout.glob
index bfb0f7753..5580dc669 100644
--- a/tests/cli-tests/basic/help.sh.stdout.glob
+++ b/tests/cli-tests/basic/help.sh.stdout.glob
@@ -1,25 +1,34 @@
 + zstd -h
-Usage: zstd *OPTION*... *FILE*... *-o file*
-Compress or uncompress FILEs (with no FILE or when FILE is `-`, read from standard input).
+Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.
 
-  -o file              result stored into `file` (only 1 output file)
-  -1 .. -19            compression level (faster .. better; default: 3)
-  -d, --decompress     decompression
-  -f, --force          disable input and output checks. Allows overwriting existing files,
-                       input from console, output to stdout, operating on links,
-                       block devices, etc. During decompression and when the output
-                       destination is stdout, pass-through unrecognized formats as-is.
-      --rm             remove source file(s) after successful de/compression
-  -k, --keep           preserve source file(s) (default)
-  -D DICT              use DICT as Dictionary for compression or decompression
-  -h                   display usage and exit
-  -H,--help            display long help and exit
+Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT*
+
+Options:
+  -o OUTPUT                     Write output to a single file, OUTPUT.
+  -k, --keep                    Preserve INPUT file(s). *Default*
+  --rm                          Remove INPUT file(s) after successful (de)compression.
+
+  -#                            Desired compression level, where `#` is a number between 1 and 19;
+                                lower numbers provide faster compression, higher numbers yield
+                                better compression ratios. *Default: 3*
+
+  -d, --decompress              Perform decompression.
+  -D DICT                       Use DICT as the dictionary for compression or decompression.
+
+  -f, --force                   Disable input and output checks. Allows overwriting existing files,
+                                receiving input from the console, printing ouput to STDOUT, and
+                                operating on links, block devices, etc. Unrecognized formats will be
+                                passed-through through as-is.
+
+  -h                            Display short usage and exit.
+  -H, --help                    Display full help and exit.
+  -V, --version                 Display the program version and exit.
 
 + zstd -H
 ...
-Advanced options :
+Advanced options:
 ...
 + zstd --help
 ...
-Advanced options :
+Advanced options:
 ...
diff --git a/tests/cli-tests/basic/memlimit.sh b/tests/cli-tests/basic/memlimit.sh
index 61a5fc084..88d734d0d 100755
--- a/tests/cli-tests/basic/memlimit.sh
+++ b/tests/cli-tests/basic/memlimit.sh
@@ -15,25 +15,25 @@ zstd --memory=32asbdf file && die "Should not allow bogus suffix"
 println "+ zstd --memory=hello file"
 zstd --memory=hello file && die "Should not allow non-numeric parameter"
 println "+ zstd --memory=1 file"
-zstd --memory=1 file && die "Should allow numeric parameter without suffix"
+zstd -q --memory=1 file && die "Should allow numeric parameter without suffix"
 rm file.zst
 println "+ zstd --memory=1K file"
-zstd --memory=1K file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1K file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 println "+ zstd --memory=1KB file"
-zstd --memory=1KB file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1KB file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 println "+ zstd --memory=1KiB file"
-zstd --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 println "+ zstd --memory=1M file"
-zstd --memory=1M file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1M file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 println "+ zstd --memory=1MB file"
-zstd --memory=1MB file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1MB file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 println "+ zstd --memory=1MiB file"
-zstd --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
+zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
 
 rm file
diff --git a/tests/cli-tests/basic/version.sh.stdout.glob b/tests/cli-tests/basic/version.sh.stdout.glob
index 54968fa41..4cc9fb9b2 100644
--- a/tests/cli-tests/basic/version.sh.stdout.glob
+++ b/tests/cli-tests/basic/version.sh.stdout.glob
@@ -1,2 +1,2 @@
-*** zstd command line interface *-bits v1.*.*, by Yann Collet ***
-*** zstd command line interface *-bits v1.*.*, by Yann Collet ***
+*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
+*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
diff --git a/tests/cli-tests/compression/adapt.sh b/tests/cli-tests/compression/adapt.sh
index 564e955b5..30b9afaa0 100755
--- a/tests/cli-tests/compression/adapt.sh
+++ b/tests/cli-tests/compression/adapt.sh
@@ -4,3 +4,11 @@ set -e
 
 # Test --adapt
 zstd -f file --adapt -c | zstd -t
+
+datagen -g100M > file100M
+
+# Pick parameters to force fast adaptation, even on slow systems
+zstd --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"
+
+# Adaption still happens with --no-progress
+zstd --no-progress --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"
diff --git a/tests/cli-tests/compression/levels.sh b/tests/cli-tests/compression/levels.sh
index 4837790ce..cc2700a30 100755
--- a/tests/cli-tests/compression/levels.sh
+++ b/tests/cli-tests/compression/levels.sh
@@ -6,29 +6,27 @@ set -v
 datagen > file
 
 # Compress with various levels and ensure that their sizes are ordered
-zstd --fast=10 file -o file-f10.zst
-zstd --fast=1 file -o file-f1.zst
-zstd -1 file -o file-1.zst
-zstd -19 file -o file-19.zst
-zstd -22 --ultra file -o file-22.zst
+zstd --fast=10 file -o file-f10.zst -q
+zstd --fast=1 file -o file-f1.zst -q
+zstd -1 file -o file-1.zst -q
+zstd -19 file -o file-19.zst -q
 
-zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
 
-cmp_size -ne file-19.zst file-22.zst
 cmp_size -lt file-19.zst file-1.zst
 cmp_size -lt file-1.zst file-f1.zst
 cmp_size -lt file-f1.zst file-f10.zst
 
 # Test default levels
-zstd --fast file -f
+zstd --fast file -f -q
 cmp file.zst file-f1.zst || die "--fast is not level -1"
 
-zstd -0 file -o file-0.zst
-zstd -f file
+zstd -0 file -o file-0.zst -q
+zstd -f file -q
 cmp file.zst file-0.zst || die "Level 0 is not the default level"
 
 # Test level clamping
-zstd -99 file -o file-99.zst
+zstd -99 file -o file-99.zst -q
 cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
 zstd --fast=200000 file -c | zstd -t
 
@@ -36,10 +34,10 @@ zstd -5000000000 -f file       && die "Level too large, must fail" ||:
 zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
 
 # Test setting a level through the environment variable
-ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst
-ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
-ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
-ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q
 
 cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
 cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
@@ -47,18 +45,18 @@ cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level
 cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
 
 # Test invalid environment clevel is the default level
-zstd -f file
-ZSTD_CLEVEL=- zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=+ zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=a zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=-a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=+a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst       ; cmp file.zst file-env.zst
-ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
+zstd -f file -q
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q    ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
 
 # Test environment clevel is overridden by command line
-ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
-ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q
 
 cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
 cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
diff --git a/tests/cli-tests/compression/levels.sh.stderr.exact b/tests/cli-tests/compression/levels.sh.stderr.exact
index cb00433e6..c8fb79c68 100644
--- a/tests/cli-tests/compression/levels.sh.stderr.exact
+++ b/tests/cli-tests/compression/levels.sh.stderr.exact
@@ -2,32 +2,31 @@
 datagen > file
 
 # Compress with various levels and ensure that their sizes are ordered
-zstd --fast=10 file -o file-f10.zst
-zstd --fast=1 file -o file-f1.zst
-zstd -1 file -o file-1.zst
-zstd -19 file -o file-19.zst
-zstd -22 --ultra file -o file-22.zst
+zstd --fast=10 file -o file-f10.zst -q
+zstd --fast=1 file -o file-f1.zst -q
+zstd -1 file -o file-1.zst -q
+zstd -19 file -o file-19.zst -q
 
-zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
+4 files decompressed : 262148 bytes total 
 
-cmp_size -ne file-19.zst file-22.zst
 cmp_size -lt file-19.zst file-1.zst
 cmp_size -lt file-1.zst file-f1.zst
 cmp_size -lt file-f1.zst file-f10.zst
 
 # Test default levels
-zstd --fast file -f
+zstd --fast file -f -q
 cmp file.zst file-f1.zst || die "--fast is not level -1"
 
-zstd -0 file -o file-0.zst
-zstd -f file
+zstd -0 file -o file-0.zst -q
+zstd -f file -q
 cmp file.zst file-0.zst || die "Level 0 is not the default level"
 
 # Test level clamping
-zstd -99 file -o file-99.zst
-Warning : compression level higher than max, reduced to 19 
+zstd -99 file -o file-99.zst -q
 cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
 zstd --fast=200000 file -c | zstd -t
+/*stdin*\           : 65537 bytes 
 
 zstd -5000000000 -f file       && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
@@ -35,11 +34,10 @@ zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
 
 # Test setting a level through the environment variable
-ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst
-ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
-ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
-ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst
-Warning : compression level higher than max, reduced to 19 
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q
 
 cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
 cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
@@ -47,25 +45,25 @@ cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level
 cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
 
 # Test invalid environment clevel is the default level
-zstd -f file
-ZSTD_CLEVEL=- zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
+zstd -f file -q
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-: not a valid integer value 
-ZSTD_CLEVEL=+ zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+: not a valid integer value 
-ZSTD_CLEVEL=a zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=a: not a valid integer value 
-ZSTD_CLEVEL=-a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-a: not a valid integer value 
-ZSTD_CLEVEL=+a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+a: not a valid integer value 
-ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst       ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q    ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=3a7: not a valid integer value 
-ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=5000000000: numeric value too large 
 
 # Test environment clevel is overridden by command line
-ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
-ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q
 
 cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
 cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
diff --git a/tests/cli-tests/compression/multi-threaded.sh b/tests/cli-tests/compression/multi-threaded.sh
index bd01448e2..17a5eb518 100755
--- a/tests/cli-tests/compression/multi-threaded.sh
+++ b/tests/cli-tests/compression/multi-threaded.sh
@@ -3,13 +3,13 @@
 set -e
 
 # Test multi-threaded flags
-zstd --single-thread file -f            ; zstd -t file.zst
-zstd -T2 -f file                        ; zstd -t file.zst
-zstd --rsyncable -f file                ; zstd -t file.zst
-zstd -T0 -f file                        ; zstd -t file.zst
-zstd -T0 --auto-threads=logical -f file ; zstd -t file.zst
-zstd -T0 --auto-threads=physical -f file; zstd -t file.zst
+zstd --single-thread file -f -q         ; zstd -t file.zst
+zstd -T2 -f file -q                     ; zstd -t file.zst
+zstd --rsyncable -f file -q             ; zstd -t file.zst
+zstd -T0 -f file -q                     ; zstd -t file.zst
+zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst
+zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst
 
 # multi-thread decompression warning test
-zstd -T0 -f file                        ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
-zstd -T0 -f file                        ; zstd -t file.zst; zstd -T2 -d file.zst -o file4
+zstd -T0 -f file -q                     ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
+zstd -T0 -f file -q                     ; zstd -t file.zst; zstd -T2 -d file.zst -o file4
diff --git a/tests/cli-tests/compression/multi-threaded.sh.stderr.exact b/tests/cli-tests/compression/multi-threaded.sh.stderr.exact
index 54d47d8bd..11daff6ba 100644
--- a/tests/cli-tests/compression/multi-threaded.sh.stderr.exact
+++ b/tests/cli-tests/compression/multi-threaded.sh.stderr.exact
@@ -1 +1,11 @@
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
 Warning : decompression does not support multi-threading
+file.zst            : 65537 bytes 
diff --git a/tests/cli-tests/compression/window-resize.sh b/tests/cli-tests/compression/window-resize.sh
new file mode 100755
index 000000000..3b5e6fe24
--- /dev/null
+++ b/tests/cli-tests/compression/window-resize.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+datagen -g1G > file
+zstd --long=31 -1 --single-thread --no-content-size -f file
+zstd -l -v file.zst
+
+# We want to ignore stderr (its outputting "*** zstd command line interface
+# 64-bits v1.5.3, by Yann Collet ***")
+
+rm file file.zst
diff --git a/tests/cli-tests/compression/window-resize.sh.stderr.ignore b/tests/cli-tests/compression/window-resize.sh.stderr.ignore
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/cli-tests/compression/window-resize.sh.stdout.glob b/tests/cli-tests/compression/window-resize.sh.stdout.glob
new file mode 100644
index 000000000..313d216e1
--- /dev/null
+++ b/tests/cli-tests/compression/window-resize.sh.stdout.glob
@@ -0,0 +1,3 @@
+...
+Window Size: 1.000 GiB (1073741824 B)
+...
diff --git a/tests/cli-tests/decompression/pass-through.sh.stderr.exact b/tests/cli-tests/decompression/pass-through.sh.stderr.exact
index f9ac13cb2..62f96ae42 100644
--- a/tests/cli-tests/decompression/pass-through.sh.stderr.exact
+++ b/tests/cli-tests/decompression/pass-through.sh.stderr.exact
@@ -1,3 +1,4 @@
+file                 :230.00%   (    10 B =>     23 B, file.zst) 
 zstd: file: unsupported format 
 zstd: file: unsupported format 
 zstd: file: unsupported format 
diff --git a/tests/cli-tests/dictionaries/dictionary-mismatch.sh b/tests/cli-tests/dictionaries/dictionary-mismatch.sh
index 8264ccca5..885cac223 100755
--- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh
+++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh
@@ -23,7 +23,7 @@ if [ false ]; then
 fi
 
 set -v
-zstd files/0 -D dicts/0
+zstd files/0 -D dicts/0 -q
 zstd -t files/0.zst -D dicts/0
 zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
 zstd -t files/0.zst            && die "Must fail" ||:
diff --git a/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact b/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
index 0afea722e..8896763c1 100644
--- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
+++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
@@ -1,5 +1,6 @@
-zstd files/0 -D dicts/0
+zstd files/0 -D dicts/0 -q
 zstd -t files/0.zst -D dicts/0
+files/0.zst         : 1000 bytes 
 zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
 files/0.zst : Decoding error (36) : Dictionary mismatch 
 zstd -t files/0.zst            && die "Must fail" ||:
diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh b/tests/cli-tests/file-stat/compress-file-to-file.sh
new file mode 100755
index 000000000..c5f590031
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+chmod 642 file
+
+zstd file -q --trace-file-stat -o file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
new file mode 100644
index 000000000..a1ad09ef7
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
@@ -0,0 +1,42 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file, file.zst)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_setFileStat(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_utime(file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat:  > UTIL_chmod(file.zst, 0642)
+Trace:FileStat:   > chmod
+Trace:FileStat:   < 0
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
diff --git a/tests/cli-tests/file-stat/compress-file-to-stdout.sh b/tests/cli-tests/file-stat/compress-file-to-stdout.sh
new file mode 100755
index 000000000..99ebfc46f
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-stdout.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd file -cq --trace-file-stat > file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..7c690d20b
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
@@ -0,0 +1,24 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-file.sh b/tests/cli-tests/file-stat/compress-stdin-to-file.sh
new file mode 100755
index 000000000..837946198
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-file.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd < file -q --trace-file-stat -o file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
new file mode 100644
index 000000000..00afd97c3
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
@@ -0,0 +1,24 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
+Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh
new file mode 100755
index 000000000..64f4b0362
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd < file -cq --trace-file-stat > file.zst
+zstd -tq file.zst
diff --git a/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..8bf05e641
--- /dev/null
+++ b/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
@@ -0,0 +1,18 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh b/tests/cli-tests/file-stat/decompress-file-to-file.sh
new file mode 100755
index 000000000..9e68f8f33
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+chmod 642 file.zst
+
+zstd -dq --trace-file-stat file.zst
diff --git a/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
new file mode 100644
index 000000000..d264c63e4
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
@@ -0,0 +1,38 @@
+Trace:FileStat: > UTIL_isLink(file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_isDirectoryStat()
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file.zst, file)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_setFileStat(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_utime(file)
+Trace:FileStat:  < 0
+Trace:FileStat:  > UTIL_chmod(file, 0642)
+Trace:FileStat:   > chmod
+Trace:FileStat:   < 0
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
diff --git a/tests/cli-tests/file-stat/decompress-file-to-stdout.sh b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh
new file mode 100755
index 000000000..518c2a950
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat file.zst > file
diff --git a/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..7fe6dda15
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
@@ -0,0 +1,18 @@
+Trace:FileStat: > UTIL_isLink(file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(file.zst)
+Trace:FileStat:  > UTIL_stat(file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_isDirectoryStat()
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-file.sh b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh
new file mode 100755
index 000000000..135d75524
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat < file.zst -o file
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
new file mode 100644
index 000000000..749fd395a
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
@@ -0,0 +1,20 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh
new file mode 100755
index 000000000..495f07b17
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat < file.zst > file
diff --git a/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
new file mode 100644
index 000000000..e36cb9d05
--- /dev/null
+++ b/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
@@ -0,0 +1,14 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(/*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(/*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
diff --git a/tests/cli-tests/progress/no-progress.sh b/tests/cli-tests/progress/no-progress.sh
new file mode 100755
index 000000000..708878f26
--- /dev/null
+++ b/tests/cli-tests/progress/no-progress.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+echo hello > hello
+echo world > world
+
+zstd -q hello world
+
+println >&2 "Tests cases where progress information should not be printed"
+
+for args in \
+	"" \
+	"--fake-stderr-is-console -q" \
+	"--fake-stderr-is-console -qq --progress" \
+	"--no-progress --fake-stderr-is-console" \
+	"--no-progress --fake-stderr-is-console -v"
+do
+	println >&2 "args = $args"
+	println >&2 "compress file to file"
+	zstd $args -f hello
+	println >&2 "compress pipe to pipe"
+	zstd $args < hello > $INTOVOID
+	println >&2 "compress pipe to file"
+	zstd $args < hello -fo hello.zst
+	println >&2 "compress file to pipe"
+	zstd $args hello -c > $INTOVOID
+	println >&2 "compress 2 files"
+	zstd $args -f hello world
+
+	println >&2 "decompress file to file"
+	zstd $args -d -f hello.zst
+	println >&2 "decompress pipe to pipe"
+	zstd $args -d < hello.zst > $INTOVOID
+	println >&2 "decompress pipe to file"
+	zstd $args -d < hello.zst -fo hello
+	println >&2 "decompress file to pipe"
+	zstd $args -d hello.zst -c > $INTOVOID
+	println >&2 "decompress 2 files"
+	zstd $args -d -f hello.zst world.zst
+	println >&2 ""
+done
diff --git a/tests/cli-tests/progress/no-progress.sh.stderr.glob b/tests/cli-tests/progress/no-progress.sh.stderr.glob
new file mode 100644
index 000000000..d0f911227
--- /dev/null
+++ b/tests/cli-tests/progress/no-progress.sh.stderr.glob
@@ -0,0 +1,96 @@
+Tests cases where progress information should not be printed
+args =
+compress file to file
+hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+2 files compressed*
+decompress file to file
+hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*
+decompress file to pipe
+decompress 2 files
+2 files decompressed*
+
+args = --fake-stderr-is-console -q
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --fake-stderr-is-console -qq --progress
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --no-progress --fake-stderr-is-console
+compress file to file
+hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+2 files compressed*
+decompress file to file
+hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*
+decompress file to pipe
+decompress 2 files
+2 files decompressed*
+
+args = --no-progress --fake-stderr-is-console -v
+compress file to file
+*Zstandard CLI*
+hello*hello.zst*
+compress pipe to pipe
+*Zstandard CLI*
+*stdin*stdout*
+compress pipe to file
+*Zstandard CLI*
+*stdin*hello.zst*
+compress file to pipe
+*Zstandard CLI*
+*hello*stdout*
+compress 2 files
+*Zstandard CLI*
+*hello*hello.zst*
+*world*world.zst*
+2 files compressed*
+decompress file to file
+*Zstandard CLI*
+hello.zst*
+decompress pipe to pipe
+*Zstandard CLI*
+*stdin*
+decompress pipe to file
+*Zstandard CLI*
+*stdin*
+decompress file to pipe
+*Zstandard CLI*
+hello.zst*
+decompress 2 files
+*Zstandard CLI*
+hello.zst*
+world.zst*
+2 files decompressed*
diff --git a/tests/cli-tests/progress/progress.sh b/tests/cli-tests/progress/progress.sh
new file mode 100755
index 000000000..eb464993a
--- /dev/null
+++ b/tests/cli-tests/progress/progress.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+println >&2 "Tests cases where progress information should be printed"
+
+echo hello > hello
+echo world > world
+
+zstd -q hello world
+
+for args in \
+	"--progress" \
+	"--fake-stderr-is-console" \
+	"--progress --fake-stderr-is-console -q"; do
+	println >&2 "args = $args"
+	println >&2 "compress file to file"
+	zstd $args -f hello
+	println >&2 "compress pipe to pipe"
+	zstd $args < hello > $INTOVOID
+	println >&2 "compress pipe to file"
+	zstd $args < hello -fo hello.zst
+	println >&2 "compress file to pipe"
+	zstd $args hello -c > $INTOVOID
+	println >&2 "compress 2 files"
+	zstd $args -f hello world
+
+	println >&2 "decompress file to file"
+	zstd $args -d -f hello.zst
+	println >&2 "decompress pipe to pipe"
+	zstd $args -d < hello.zst > $INTOVOID
+	println >&2 "decompress pipe to file"
+	zstd $args -d < hello.zst -fo hello
+	println >&2 "decompress file to pipe"
+	zstd $args -d hello.zst -c > $INTOVOID
+	println >&2 "decompress 2 files"
+	zstd $args -d -f hello.zst world.zst
+	println >&2 ""
+done
diff --git a/tests/cli-tests/progress/progress.sh.stderr.glob b/tests/cli-tests/progress/progress.sh.stderr.glob
new file mode 100644
index 000000000..ca620d3c2
--- /dev/null
+++ b/tests/cli-tests/progress/progress.sh.stderr.glob
@@ -0,0 +1,62 @@
+Tests cases where progress information should be printed
+args = --progress
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+*Read:*stdin*stdout*
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+*Read:*hello*stdout*
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+*stdin*stdin*
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+*hello.zst*hello.zst*
+decompress 2 files
+*hello.zst*2 files decompressed*
+
+args = --fake-stderr-is-console
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+decompress 2 files
+*hello.zst*2 files decompressed*
+
+args = --progress --fake-stderr-is-console -q
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+*Read:*stdin*stdout*
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+*Read:*hello*stdout*
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+*stdin*stdin*
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+*hello.zst*hello.zst*
+decompress 2 files
+*hello.zst*2 files decompressed*
diff --git a/tests/cli-tests/run.py b/tests/cli-tests/run.py
index d726fba7f..45af5124b 100755
--- a/tests/cli-tests/run.py
+++ b/tests/cli-tests/run.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -209,6 +209,7 @@ class Options:
         preserve: bool,
         scratch_dir: str,
         test_dir: str,
+        set_exact_output: bool,
     ) -> None:
         self.env = env
         self.timeout = timeout
@@ -216,6 +217,7 @@ class Options:
         self.preserve = preserve
         self.scratch_dir = scratch_dir
         self.test_dir = test_dir
+        self.set_exact_output = set_exact_output
 
 
 class TestCase:
@@ -335,7 +337,7 @@ class TestCase:
             self._test_stdin.close()
             self._test_stdin = None
 
-    def _check_output_exact(self, out_name: str, expected: bytes) -> None:
+    def _check_output_exact(self, out_name: str, expected: bytes, exact_name: str) -> None:
         """
         Check the output named :out_name: for an exact match against the :expected: content.
         Saves the success and message.
@@ -349,6 +351,10 @@ class TestCase:
             self._success[check_name] = False
             self._message[check_name] = f"{out_name} does not match!\n> diff expected actual\n{diff(expected, actual)}"
 
+            if self._opts.set_exact_output:
+                with open(exact_name, "wb") as f:
+                    f.write(actual)
+
     def _check_output_glob(self, out_name: str, expected: bytes) -> None:
         """
         Check the output named :out_name: for a glob match against the :expected: glob.
@@ -386,15 +392,13 @@ class TestCase:
         ignore_name = f"{self._test_file}.{out_name}.ignore"
 
         if os.path.exists(exact_name):
-            return self._check_output_exact(out_name, read_file(exact_name))
+            return self._check_output_exact(out_name, read_file(exact_name), exact_name)
         elif os.path.exists(glob_name):
             return self._check_output_glob(out_name, read_file(glob_name))
-        elif os.path.exists(ignore_name):
+        else:
             check_name = f"check_{out_name}"
             self._success[check_name] = True
             self._message[check_name] = f"{out_name} ignored!"
-        else:
-            return self._check_output_exact(out_name, bytes())
 
     def _check_stderr(self) -> None:
         """Checks the stderr output against the expectation."""
@@ -678,6 +682,11 @@ if __name__ == "__main__":
             "Scratch directory located in TEST_DIR/scratch/."
         )
     )
+    parser.add_argument(
+        "--set-exact-output",
+        action="store_true",
+        help="Set stderr.exact and stdout.exact for all failing tests, unless .ignore or .glob already exists"
+    )
     parser.add_argument(
         "tests",
         nargs="*",
@@ -714,6 +723,7 @@ if __name__ == "__main__":
         preserve=args.preserve,
         test_dir=args.test_dir,
         scratch_dir=scratch_dir,
+        set_exact_output=args.set_exact_output,
     )
 
     if len(args.tests) == 0:
@@ -726,4 +736,3 @@ if __name__ == "__main__":
         sys.exit(0)
     else:
         sys.exit(1)
-
diff --git a/tests/datagencli.c b/tests/datagencli.c
index 7300fdb76..09ec5e9ae 100644
--- a/tests/datagencli.c
+++ b/tests/datagencli.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index d2b126f39..148088a76 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,6 +14,7 @@
 #include 
 #include 
 #include 
+#include   /* time(), for seed random initialization */
 
 #include "util.h"
 #include "timefn.h"   /* UTIL_clockSpanMicro, SEC_TO_MICRO, UTIL_TIME_INITIALIZER */
@@ -24,21 +25,13 @@
 #include "zdict.h"
 
 /* Direct access to internal compression functions is required */
-#include "zstd_compress.c"
+#include "compress/zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
 
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"     /* XXH64 */
 
-#ifndef MIN
-    #define MIN(a, b) ((a) < (b) ? (a) : (b))
-#endif
-
-#ifndef MAX_PATH
-    #ifdef PATH_MAX
-        #define MAX_PATH PATH_MAX
-    #else
-        #define MAX_PATH 256
-    #endif
+#if !(defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */))
+# define inline  /* disable */
 #endif
 
 /*-************************************
@@ -70,6 +63,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
         }                                                                      \
     } while (0)
 
+
 /*-*******************************************************
 *  Random function
 *********************************************************/
@@ -136,7 +130,7 @@ static void RAND_genDist(U32* seed, BYTE* dist, double weight)
     BYTE step = (BYTE) ((RAND(seed) % 256) | 1); /* force it to be odd so it's relatively prime to 256 */
 
     while (i < DISTSIZE) {
-        size_t states = ((size_t)(weight * statesLeft)) + 1;
+        size_t states = ((size_t)(weight * (double)statesLeft)) + 1;
         size_t j;
         for (j = 0; j < states && i < DISTSIZE; j++, i++) {
             dist[i] = symb;
@@ -165,7 +159,7 @@ static double RAND_exp(U32* seed, double mean)
 /*-*******************************************************
 *  Constants and Structs
 *********************************************************/
-const char *BLOCK_TYPES[] = {"raw", "rle", "compressed"};
+const char* BLOCK_TYPES[] = {"raw", "rle", "compressed"};
 
 #define MAX_DECOMPRESSED_SIZE_LOG 20
 #define MAX_DECOMPRESSED_SIZE (1ULL << MAX_DECOMPRESSED_SIZE_LOG)
@@ -175,6 +169,14 @@ const char *BLOCK_TYPES[] = {"raw", "rle", "compressed"};
 #define MIN_SEQ_LEN (3)
 #define MAX_NB_SEQ ((ZSTD_BLOCKSIZE_MAX + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN)
 
+#ifndef MAX_PATH
+    #ifdef PATH_MAX
+        #define MAX_PATH PATH_MAX
+    #else
+        #define MAX_PATH 256
+    #endif
+#endif
+
 BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE];
 BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2];
 BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX];
@@ -240,6 +242,10 @@ typedef enum {
   gt_block,      /* generate compressed blocks without block/frame headers */
 } genType_e;
 
+#ifndef MIN
+    #define MIN(a, b) ((a) < (b) ? (a) : (b))
+#endif
+
 /*-*******************************************************
 *  Global variables (set from command line)
 *********************************************************/
@@ -454,7 +460,7 @@ static size_t writeHufHeader(U32* seed, HUF_CElt* hufTable, void* dst, size_t ds
     }
 
     /* Write table description header */
-    {   size_t const hSize = HUF_writeCTable (op, dstSize, hufTable, maxSymbolValue, huffLog);
+    {   size_t const hSize = HUF_writeCTable_wksp (op, dstSize, hufTable, maxSymbolValue, huffLog, WKSP, sizeof(WKSP));
         if (hSize + 12 >= srcSize) return 0;   /* not useful to try compression */
         op += hSize;
     }
@@ -558,10 +564,10 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
                     sizeFormat == 0
                             ? HUF_compress1X_usingCTable(
                                       op, opend - op, LITERAL_BUFFER, litSize,
-                                      frame->stats.hufTable)
+                                      frame->stats.hufTable, /* flags */ 0)
                             : HUF_compress4X_usingCTable(
                                       op, opend - op, LITERAL_BUFFER, litSize,
-                                      frame->stats.hufTable);
+                                      frame->stats.hufTable, /* flags */ 0);
             CHECKERR(compressedSize);
             /* this only occurs when it could not compress or similar */
         } while (compressedSize <= 0);
@@ -662,11 +668,11 @@ generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
          * ensure nice numbers */
         U32 matchLen =
                 MIN_SEQ_LEN +
-                ROUND(RAND_exp(seed, excessMatch / (double)(numSequences - i)));
+                ROUND(RAND_exp(seed, (double)excessMatch / (double)(numSequences - i)));
         U32 literalLen =
                 (RAND(seed) & 7)
                         ? ROUND(RAND_exp(seed,
-                                         literalsSize /
+                                         (double)literalsSize /
                                                  (double)(numSequences - i)))
                         : 0;
         /* actual offset, code to send, and point to copy up to when shifting
diff --git a/tests/external_matchfinder.c b/tests/external_matchfinder.c
new file mode 100644
index 000000000..97c47caff
--- /dev/null
+++ b/tests/external_matchfinder.c
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "external_matchfinder.h"
+#include 
+#include "zstd_compress_internal.h"
+
+#define HSIZE 1024
+static U32 const HLOG = 10;
+static U32 const MLS = 4;
+static U32 const BADIDX = 0xffffffff;
+
+static size_t simpleExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+) {
+    const BYTE* const istart = (const BYTE*)src;
+    const BYTE* const iend = istart + srcSize;
+    const BYTE* ip = istart;
+    const BYTE* anchor = istart;
+    size_t seqCount = 0;
+    U32 hashTable[HSIZE];
+
+    (void)externalMatchState;
+    (void)dict;
+    (void)dictSize;
+    (void)outSeqsCapacity;
+    (void)compressionLevel;
+
+    {   int i;
+        for (i=0; i < HSIZE; i++) {
+            hashTable[i] = BADIDX;
+    }   }
+
+    while (ip + MLS < iend) {
+        size_t const hash = ZSTD_hashPtr(ip, HLOG, MLS);
+        U32 const matchIndex = hashTable[hash];
+        hashTable[hash] = (U32)(ip - istart);
+
+        if (matchIndex != BADIDX) {
+            const BYTE* const match = istart + matchIndex;
+            U32 const matchLen = (U32)ZSTD_count(ip, match, iend);
+            if (matchLen >= ZSTD_MINMATCH_MIN) {
+                U32 const litLen = (U32)(ip - anchor);
+                U32 const offset = (U32)(ip - match);
+                ZSTD_Sequence const seq = {
+                    offset, litLen, matchLen, 0
+                };
+
+                /* Note: it's crucial to stay within the window size! */
+                if (offset <= windowSize) {
+                    outSeqs[seqCount++] = seq;
+                    ip += matchLen;
+                    anchor = ip;
+                    continue;
+                }
+            }
+        }
+
+        ip++;
+    }
+
+    {   ZSTD_Sequence const finalSeq = {
+            0, (U32)(iend - anchor), 0, 0
+        };
+        outSeqs[seqCount++] = finalSeq;
+    }
+
+    return seqCount;
+}
+
+size_t zstreamExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+) {
+    EMF_testCase const testCase = *((EMF_testCase*)externalMatchState);
+    memset(outSeqs, 0, outSeqsCapacity);
+
+    switch (testCase) {
+        case EMF_ZERO_SEQS:
+            return 0;
+        case EMF_ONE_BIG_SEQ:
+            outSeqs[0].offset = 0;
+            outSeqs[0].matchLength = 0;
+            outSeqs[0].litLength = (U32)(srcSize);
+            return 1;
+         case EMF_LOTS_OF_SEQS:
+            return simpleExternalMatchFinder(
+                externalMatchState,
+                outSeqs, outSeqsCapacity,
+                src, srcSize,
+                dict, dictSize,
+                compressionLevel,
+                windowSize
+            );
+        case EMF_INVALID_OFFSET:
+            outSeqs[0].offset = 1 << 20;
+            outSeqs[0].matchLength = 4;
+            outSeqs[0].litLength = (U32)(srcSize - 4);
+            return 1;
+        case EMF_INVALID_MATCHLEN:
+            outSeqs[0].offset = 1;
+            outSeqs[0].matchLength = (U32)(srcSize);
+            outSeqs[0].litLength = 1;
+            return 1;
+        case EMF_INVALID_LITLEN:
+            outSeqs[0].offset = 0;
+            outSeqs[0].matchLength = 0;
+            outSeqs[0].litLength = (U32)(srcSize + 1);
+            return 1;
+        case EMF_INVALID_LAST_LITS:
+            outSeqs[0].offset = 1;
+            outSeqs[0].matchLength = 1;
+            outSeqs[0].litLength = 1;
+            outSeqs[1].offset = 0;
+            outSeqs[1].matchLength = 0;
+            outSeqs[1].litLength = (U32)(srcSize - 1);
+            return 2;
+        case EMF_SMALL_ERROR:
+            return outSeqsCapacity + 1;
+        case EMF_BIG_ERROR:
+        default:
+            return ZSTD_EXTERNAL_MATCHFINDER_ERROR;
+    }
+}
diff --git a/tests/external_matchfinder.h b/tests/external_matchfinder.h
new file mode 100644
index 000000000..7550bbceb
--- /dev/null
+++ b/tests/external_matchfinder.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef EXTERNAL_MATCHFINDER
+#define EXTERNAL_MATCHFINDER
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+/* See external_matchfinder.c for details on each test case */
+typedef enum {
+    EMF_ZERO_SEQS = 0,
+    EMF_ONE_BIG_SEQ = 1,
+    EMF_LOTS_OF_SEQS = 2,
+    EMF_BIG_ERROR = 3,
+    EMF_SMALL_ERROR = 4,
+    EMF_INVALID_OFFSET = 5,
+    EMF_INVALID_MATCHLEN = 6,
+    EMF_INVALID_LITLEN = 7,
+    EMF_INVALID_LAST_LITS = 8
+} EMF_testCase;
+
+size_t zstreamExternalMatchFinder(
+  void* externalMatchState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+#endif /* EXTERNAL_MATCHFINDER */
diff --git a/tests/fullbench.c b/tests/fullbench.c
index b55ff767f..3a72d89d3 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -171,6 +171,7 @@ FORCE_NOINLINE size_t ZSTD_decodeLiteralsHeader(ZSTD_DCtx* dctx, void const* src
                 size_t lhSize, litSize, litCSize;
                 U32 const lhlCode = (istart[0] >> 2) & 3;
                 U32 const lhc = MEM_readLE32(istart);
+                int const flags = ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0;
                 switch(lhlCode)
                 {
                 case 0: case 1: default:   /* note : default is impossible, since lhlCode into [0..3] */
@@ -195,16 +196,16 @@ FORCE_NOINLINE size_t ZSTD_decodeLiteralsHeader(ZSTD_DCtx* dctx, void const* src
                 RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
                 RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
 #ifndef HUF_FORCE_DECOMPRESS_X2
-                return HUF_readDTableX1_wksp_bmi2(
+                return HUF_readDTableX1_wksp(
                         dctx->entropy.hufTable,
                         istart+lhSize, litCSize,
                         dctx->workspace, sizeof(dctx->workspace),
-                        ZSTD_DCtx_get_bmi2(dctx));
+                        flags);
 #else
                 return HUF_readDTableX2_wksp(
                         dctx->entropy.hufTable,
                         istart+lhSize, litCSize,
-                        dctx->workspace, sizeof(dctx->workspace));
+                        dctx->workspace, sizeof(dctx->workspace), flags);
 #endif
             }
         }
diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile
index 912348c38..4b5102ef1 100644
--- a/tests/fuzz/Makefile
+++ b/tests/fuzz/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -34,9 +34,14 @@ ZSTDDIR = ../../lib
 PRGDIR = ../../programs
 CONTRIBDIR = ../../contrib
 
+# TODO(embg) make it possible to plug in an arbitrary matchfinder as a .o file
+MATCHFINDER_DIR = $(CONTRIBDIR)/externalMatchfinder
+MATCHFINDER_SRC = $(MATCHFINDER_DIR)/matchfinder.c
+
 FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
 	-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \
-	-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
+	-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -I$(MATCHFINDER_DIR) \
+	-DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
 FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
 	-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
 	-Wstrict-prototypes -Wundef \
@@ -69,7 +74,8 @@ FUZZ_SRC       := \
 	$(ZSTDCOMMON_SRC) \
 	$(ZSTDCOMP_SRC) \
 	$(ZSTDDICT_SRC) \
-	$(ZSTDLEGACY_SRC)
+	$(ZSTDLEGACY_SRC) \
+	$(MATCHFINDER_SRC)
 FUZZ_SRC := $(sort $(wildcard $(FUZZ_SRC)))
 
 FUZZ_D_OBJ1 := $(subst $(ZSTDDIR)/common/,d_lib_common_,$(FUZZ_SRC))
@@ -78,9 +84,10 @@ FUZZ_D_OBJ3 := $(subst $(ZSTDDIR)/decompress/,d_lib_decompress_,$(FUZZ_D_OBJ2))
 FUZZ_D_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,d_lib_dictBuilder_,$(FUZZ_D_OBJ3))
 FUZZ_D_OBJ5 := $(subst $(ZSTDDIR)/legacy/,d_lib_legacy_,$(FUZZ_D_OBJ4))
 FUZZ_D_OBJ6 := $(subst $(PRGDIR)/,d_prg_,$(FUZZ_D_OBJ5))
-FUZZ_D_OBJ7 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ6))
-FUZZ_D_OBJ8 := $(FUZZ_D_OBJ7:.c=.o)
-FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ8:.S=.o)
+FUZZ_D_OBJ7 := $(subst $(MATCHFINDER_DIR)/,d_matchfinder_,$(FUZZ_D_OBJ6))
+FUZZ_D_OBJ8 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ7))
+FUZZ_D_OBJ9 := $(FUZZ_D_OBJ8:.c=.o)
+FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ9:.S=.o)
 
 FUZZ_RT_OBJ1 := $(subst $(ZSTDDIR)/common/,rt_lib_common_,$(FUZZ_SRC))
 FUZZ_RT_OBJ2 := $(subst $(ZSTDDIR)/compress/,rt_lib_compress_,$(FUZZ_RT_OBJ1))
@@ -88,9 +95,10 @@ FUZZ_RT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,rt_lib_decompress_,$(FUZZ_RT_OBJ2
 FUZZ_RT_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,rt_lib_dictBuilder_,$(FUZZ_RT_OBJ3))
 FUZZ_RT_OBJ5 := $(subst $(ZSTDDIR)/legacy/,rt_lib_legacy_,$(FUZZ_RT_OBJ4))
 FUZZ_RT_OBJ6 := $(subst $(PRGDIR)/,rt_prg_,$(FUZZ_RT_OBJ5))
-FUZZ_RT_OBJ7 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ6))
-FUZZ_RT_OBJ8 := $(FUZZ_RT_OBJ7:.c=.o)
-FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ8:.S=.o)
+FUZZ_RT_OBJ7 := $(subst $(MATCHFINDER_DIR)/,rt_matchfinder_,$(FUZZ_RT_OBJ6))
+FUZZ_RT_OBJ8 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ7))
+FUZZ_RT_OBJ9 := $(FUZZ_RT_OBJ8:.c=.o)
+FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ9:.S=.o)
 
 .PHONY: default all clean cleanall
 
@@ -143,6 +151,9 @@ rt_prg_%.o: $(PRGDIR)/%.c
 rt_fuzz_%.o: %.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
 
+rt_matchfinder_%.o: $(MATCHFINDER_DIR)/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
 d_lib_common_%.o: $(ZSTDDIR)/common/%.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
 
@@ -167,6 +178,9 @@ d_prg_%.o: $(PRGDIR)/%.c
 d_fuzz_%.o: %.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
 
+d_matchfinder_%.o: $(MATCHFINDER_DIR)/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
+
 simple_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o
 	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
 
diff --git a/tests/fuzz/block_decompress.c b/tests/fuzz/block_decompress.c
index bdbf76916..e4767b3f5 100644
--- a/tests/fuzz/block_decompress.c
+++ b/tests/fuzz/block_decompress.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c
index 46a84c7b9..54012a1da 100644
--- a/tests/fuzz/block_round_trip.c
+++ b/tests/fuzz/block_round_trip.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/decompress_dstSize_tooSmall.c b/tests/fuzz/decompress_dstSize_tooSmall.c
index 3f7607b29..8ad3ea1c9 100644
--- a/tests/fuzz/decompress_dstSize_tooSmall.c
+++ b/tests/fuzz/decompress_dstSize_tooSmall.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c
index 33c58c8bd..8dea3e560 100644
--- a/tests/fuzz/dictionary_decompress.c
+++ b/tests/fuzz/dictionary_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_loader.c b/tests/fuzz/dictionary_loader.c
index 5b60bc484..1ac22740f 100644
--- a/tests/fuzz/dictionary_loader.c
+++ b/tests/fuzz/dictionary_loader.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c
index 0b20e8d67..6dd78c3f3 100644
--- a/tests/fuzz/dictionary_round_trip.c
+++ b/tests/fuzz/dictionary_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/dictionary_stream_round_trip.c b/tests/fuzz/dictionary_stream_round_trip.c
index 9af712f0e..3ebbd843b 100644
--- a/tests/fuzz/dictionary_stream_round_trip.c
+++ b/tests/fuzz/dictionary_stream_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fse_read_ncount.c b/tests/fuzz/fse_read_ncount.c
index c32386099..29e19442a 100644
--- a/tests/fuzz/fse_read_ncount.c
+++ b/tests/fuzz/fse_read_ncount.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz.h b/tests/fuzz/fuzz.h
index 810daa2ce..d1e439fef 100644
--- a/tests/fuzz/fuzz.h
+++ b/tests/fuzz/fuzz.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -26,8 +26,7 @@
  * @param MEM_FORCE_MEMORY_ACCESS:
  *        This flag controls how the zstd library accesses unaligned memory.
  *        It can be undefined, or 0 through 2. If it is undefined, it selects
- *        the method to use based on the compiler. If testing with UBSAN set
- *        MEM_FORCE_MEMORY_ACCESS=0 to use the standard compliant method.
+ *        the method to use based on the compiler.
  * @param FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
  *        This is the canonical flag to enable deterministic builds for fuzzing.
  *        Changes to zstd for fuzzing are gated behind this define.
diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index 17eac4f6f..03ffeeefd 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c
index eae8ee4b6..a93e8ba95 100644
--- a/tests/fuzz/fuzz_data_producer.c
+++ b/tests/fuzz/fuzz_data_producer.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h
index 62771a9f8..8ca501f9b 100644
--- a/tests/fuzz/fuzz_data_producer.h
+++ b/tests/fuzz/fuzz_data_producer.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_helpers.c b/tests/fuzz/fuzz_helpers.c
index 61c0debe5..1b6ad9736 100644
--- a/tests/fuzz/fuzz_helpers.c
+++ b/tests/fuzz/fuzz_helpers.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/fuzz_helpers.h b/tests/fuzz/fuzz_helpers.h
index c180478e9..aaf4c1df4 100644
--- a/tests/fuzz/fuzz_helpers.h
+++ b/tests/fuzz/fuzz_helpers.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/huf_decompress.c b/tests/fuzz/huf_decompress.c
index fea09fc93..fcd4b1a3b 100644
--- a/tests/fuzz/huf_decompress.c
+++ b/tests/fuzz/huf_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -13,8 +13,6 @@
  * compares the result with the original, and calls abort() on corruption.
  */
 
-#define HUF_STATIC_LINKING_ONLY
-
 #include 
 #include 
 #include 
@@ -30,7 +28,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     /* Select random parameters: #streams, X1 or X2 decoding, bmi2 */
     int const streams = FUZZ_dataProducer_int32Range(producer, 0, 1);
     int const symbols = FUZZ_dataProducer_int32Range(producer, 0, 1);
-    int const bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1);
+    int const flags = 0
+        | (ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_bmi2 : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_optimalDepth : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_preferRepeat : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_suspectUncompressible : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableAsm : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableFast : 0);
     /* Select a random cBufSize - it may be too small */
     size_t const dBufSize = FUZZ_dataProducer_uint32Range(producer, 0, 8 * size + 500);
     size_t const maxTableLog = FUZZ_dataProducer_uint32Range(producer, 1, HUF_TABLELOG_MAX);
@@ -42,18 +46,18 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     size = FUZZ_dataProducer_remainingBytes(producer);
 
     if (symbols == 0) {
-        size_t const err = HUF_readDTableX1_wksp_bmi2(dt, src, size, wksp, wkspSize, bmi2);
+        size_t const err = HUF_readDTableX1_wksp(dt, src, size, wksp, wkspSize, flags);
         if (ZSTD_isError(err))
             goto _out;
     } else {
-        size_t const err = HUF_readDTableX2_wksp_bmi2(dt, src, size, wksp, wkspSize, bmi2);
+        size_t const err = HUF_readDTableX2_wksp(dt, src, size, wksp, wkspSize, flags);
         if (ZSTD_isError(err))
             goto _out;
     }
     if (streams == 0)
-        HUF_decompress1X_usingDTable_bmi2(dBuf, dBufSize, src, size, dt, bmi2);
+        HUF_decompress1X_usingDTable(dBuf, dBufSize, src, size, dt, flags);
     else
-        HUF_decompress4X_usingDTable_bmi2(dBuf, dBufSize, src, size, dt, bmi2);
+        HUF_decompress4X_usingDTable(dBuf, dBufSize, src, size, dt, flags);
 
 _out:
     free(dt);
diff --git a/tests/fuzz/huf_round_trip.c b/tests/fuzz/huf_round_trip.c
index 32b08d6d5..4d0f8de23 100644
--- a/tests/fuzz/huf_round_trip.c
+++ b/tests/fuzz/huf_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -13,8 +13,6 @@
  * compares the result with the original, and calls abort() on corruption.
  */
 
-#define HUF_STATIC_LINKING_ONLY
-
 #include 
 #include 
 #include 
@@ -46,7 +44,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     /* Select random parameters: #streams, X1 or X2 decoding, bmi2 */
     int const streams = FUZZ_dataProducer_int32Range(producer, 0, 1);
     int const symbols = FUZZ_dataProducer_int32Range(producer, 0, 1);
-    int const bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1);
+    int const flags = 0
+        | (ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_bmi2 : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_optimalDepth : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_preferRepeat : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_suspectUncompressible : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableAsm : 0)
+        | (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableFast : 0);
     /* Select a random cBufSize - it may be too small */
     size_t const cBufSize = FUZZ_dataProducer_uint32Range(producer, 0, 4 * size);
     /* Select a random tableLog - we'll adjust it up later */
@@ -83,9 +87,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     HUF_DTable* dt = (HUF_DTable*)FUZZ_malloc(HUF_DTABLE_SIZE(tableLog) * sizeof(HUF_DTable));
     dt[0] = tableLog * 0x01000001;
 
-    HUF_depth_mode depthMode = rand() & 1 ? HUF_depth_fast : HUF_depth_optimal;
-
-    tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol, wksp, wkspSize, ct, count, depthMode);
+    tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol, wksp, wkspSize, ct, count, flags);
     FUZZ_ASSERT(tableLog <= 12);
     tableLog = HUF_buildCTable_wksp(ct, count, maxSymbol, tableLog, wksp, wkspSize);
     FUZZ_ZASSERT(tableLog);
@@ -96,11 +98,11 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     }
     FUZZ_ZASSERT(tableSize);
     if (symbols == 0) {
-        FUZZ_ZASSERT(HUF_readDTableX1_wksp_bmi2(dt, cBuf, tableSize, wksp, wkspSize, bmi2));
+        FUZZ_ZASSERT(HUF_readDTableX1_wksp(dt, cBuf, tableSize, wksp, wkspSize, flags));
     } else {
-        size_t const ret = HUF_readDTableX2_wksp(dt, cBuf, tableSize, wksp, wkspSize);
+        size_t const ret = HUF_readDTableX2_wksp(dt, cBuf, tableSize, wksp, wkspSize, flags);
         if (ERR_getErrorCode(ret) == ZSTD_error_tableLog_tooLarge) {
-            FUZZ_ZASSERT(HUF_readDTableX1_wksp_bmi2(dt, cBuf, tableSize, wksp, wkspSize, bmi2));
+            FUZZ_ZASSERT(HUF_readDTableX1_wksp(dt, cBuf, tableSize, wksp, wkspSize, flags));
         } else {
             FUZZ_ZASSERT(ret);
         }
@@ -109,15 +111,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     size_t cSize;
     size_t rSize;
     if (streams == 0) {
-        cSize = HUF_compress1X_usingCTable_bmi2(cBuf, cBufSize, src, size, ct, bmi2);
+        cSize = HUF_compress1X_usingCTable(cBuf, cBufSize, src, size, ct, flags);
         FUZZ_ZASSERT(cSize);
         if (cSize != 0)
-            rSize = HUF_decompress1X_usingDTable_bmi2(rBuf, size, cBuf, cSize, dt, bmi2);
+            rSize = HUF_decompress1X_usingDTable(rBuf, size, cBuf, cSize, dt, flags);
     } else {
-        cSize = HUF_compress4X_usingCTable_bmi2(cBuf, cBufSize, src, size, ct, bmi2);
+        cSize = HUF_compress4X_usingCTable(cBuf, cBufSize, src, size, ct, flags);
         FUZZ_ZASSERT(cSize);
         if (cSize != 0)
-            rSize = HUF_decompress4X_usingDTable_bmi2(rBuf, size, cBuf, cSize, dt, bmi2);
+            rSize = HUF_decompress4X_usingDTable(rBuf, size, cBuf, cSize, dt, flags);
     }
     if (cSize != 0) {
         FUZZ_ZASSERT(rSize);
diff --git a/tests/fuzz/raw_dictionary_round_trip.c b/tests/fuzz/raw_dictionary_round_trip.c
index 0e6517654..7ceab2be6 100644
--- a/tests/fuzz/raw_dictionary_round_trip.c
+++ b/tests/fuzz/raw_dictionary_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/regression_driver.c b/tests/fuzz/regression_driver.c
index e6d2dec64..550c65d86 100644
--- a/tests/fuzz/regression_driver.c
+++ b/tests/fuzz/regression_driver.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/seekable_roundtrip.c b/tests/fuzz/seekable_roundtrip.c
index dcdcaae11..6f0aa288f 100644
--- a/tests/fuzz/seekable_roundtrip.c
+++ b/tests/fuzz/seekable_roundtrip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/sequence_compression_api.c b/tests/fuzz/sequence_compression_api.c
index cc872a081..9d3f0a18a 100644
--- a/tests/fuzz/sequence_compression_api.c
+++ b/tests/fuzz/sequence_compression_api.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -32,10 +32,14 @@ static void* literalsBuffer = NULL;
 static void* generatedSrc = NULL;
 static ZSTD_Sequence* generatedSequences = NULL;
 
+static void* dictBuffer = NULL;
+static ZSTD_CDict* cdict = NULL;
+static ZSTD_DDict* ddict = NULL;
+
 #define ZSTD_FUZZ_GENERATED_SRC_MAXSIZE (1 << 20) /* Allow up to 1MB generated data */
+#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 20) /* Fixed size 1MB literals buffer */
 #define ZSTD_FUZZ_MATCHLENGTH_MAXSIZE (1 << 18) /* Allow up to 256KB matches */
-#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << 18) /* Allow up to a 256KB dict */
-#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 18) /* Fixed size 256KB literals buffer */
+#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << ZSTD_WINDOWLOG_MAX_32) /* Allow up to 1 << ZSTD_WINDOWLOG_MAX_32 dictionary */
 #define ZSTD_FUZZ_MAX_NBSEQ (1 << 17) /* Maximum of 128K sequences */
 
 /* Deterministic random number generator */
@@ -55,9 +59,9 @@ static uint32_t FUZZ_RDG_rand(uint32_t* src)
 /* Make a pseudorandom string - this simple function exists to avoid
  * taking a dependency on datagen.h to have RDG_genBuffer().
  */
-static char* generatePseudoRandomString(char* str, size_t size) {
+static char* generatePseudoRandomString(char* str, size_t size, FUZZ_dataProducer_t* producer) {
     const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_";
-    uint32_t seed = 0;
+    uint32_t seed = FUZZ_dataProducer_uint32(producer);
     if (size) {
         for (size_t n = 0; n < size; n++) {
             int key = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1);
@@ -100,14 +104,14 @@ static size_t decodeSequences(void* dst, size_t nbSequences,
             size_t j = 0;
             size_t k = 0;
             if (dictSize != 0) {
-                if (generatedSequences[i].offset > bytesWritten) {
-                    /* Offset goes into the dictionary */
-                    size_t offsetFromEndOfDict = generatedSequences[i].offset - bytesWritten;
-                    for (; k < offsetFromEndOfDict && k < matchLength; ++k) {
-                        op[k] = dictPtr[dictSize - offsetFromEndOfDict + k];
+                if (generatedSequences[i].offset > bytesWritten) { /* Offset goes into the dictionary */
+                    size_t dictOffset = generatedSequences[i].offset - bytesWritten;
+                    size_t matchInDict = MIN(matchLength, dictOffset);
+                    for (; k < matchInDict; ++k) {
+                        op[k] = dictPtr[dictSize - dictOffset + k];
                     }
-                    matchLength -= k;
-                    op += k;
+                    matchLength -= matchInDict;
+                    op += matchInDict;
                 }
             }
             for (; j < matchLength; ++j) {
@@ -139,8 +143,8 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
                                       size_t windowLog, ZSTD_sequenceFormat_e mode)
 {
     const uint32_t repCode = 0;  /* not used by sequence ingestion api */
-    const uint32_t windowSize = 1 << windowLog;
-    const uint32_t blockSizeMax = MIN(128 << 10, 1 << windowLog);
+    size_t windowSize = 1ULL << windowLog;
+    size_t blockSizeMax = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
     uint32_t matchLengthMax = ZSTD_FUZZ_MATCHLENGTH_MAXSIZE;
     uint32_t bytesGenerated = 0;
     uint32_t nbSeqGenerated = 0;
@@ -210,38 +214,31 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
             }
             generatedSequences[nbSeqGenerated++] = seq;
             isFirstSequence = 0;
-    }   }
+        }
+    }
+
     if (mode == ZSTD_sf_explicitBlockDelimiters) {
         /* always end sequences with a block delimiter */
         const ZSTD_Sequence endBlock = {0, 0, 0, 0};
         assert(nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ);
         generatedSequences[nbSeqGenerated++] = endBlock;
     }
-
     return nbSeqGenerated;
 }
 
 static size_t roundTripTest(void* result, size_t resultCapacity,
                             void* compressed, size_t compressedCapacity,
                             const void* src, size_t srcSize,
-                            const void* dict, size_t dictSize,
                             const ZSTD_Sequence* seqs, size_t seqSize,
-                            int wLog, int cLevel, unsigned hasDict,
+                            unsigned hasDict,
                             ZSTD_sequenceFormat_e mode)
 {
     size_t cSize;
     size_t dSize;
 
-    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
     if (hasDict) {
-        FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary(cctx, dict, dictSize));
-        FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary(dctx, dict, dictSize));
+        FUZZ_ZASSERT(ZSTD_CCtx_refCDict(cctx, cdict));
+        FUZZ_ZASSERT(ZSTD_DCtx_refDDict(dctx, ddict));
     }
 
     cSize = ZSTD_compressSequences(cctx, compressed, compressedCapacity,
@@ -272,7 +269,6 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
     size_t cBufSize;
     size_t generatedSrcSize;
     size_t nbSequences;
-    void* dictBuffer = NULL;
     size_t dictSize = 0;
     unsigned hasDict;
     unsigned wLog;
@@ -281,23 +277,62 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
 
     FUZZ_dataProducer_t* const producer = FUZZ_dataProducer_create(src, size);
     FUZZ_ASSERT(producer);
-    if (literalsBuffer == NULL) {
+
+    if (!cctx) {
+        cctx = ZSTD_createCCtx();
+        FUZZ_ASSERT(cctx);
+    }
+    if (!dctx) {
+        dctx = ZSTD_createDCtx();
+        FUZZ_ASSERT(dctx);
+    }
+
+    /* Generate window log first so we don't generate offsets too large */
+    wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
+    cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
+    mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
+
+    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach);
+
+    if (!literalsBuffer) {
         literalsBuffer = FUZZ_malloc(ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
         FUZZ_ASSERT(literalsBuffer);
-        literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
+        literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, producer);
     }
 
+    if (!dictBuffer) { /* Generate global dictionary buffer */
+        ZSTD_compressionParameters cParams;
+
+        /* Generate a large dictionary buffer */
+        dictBuffer = calloc(ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, 1);
+        FUZZ_ASSERT(dictBuffer);
+
+        /* Create global cdict and ddict */
+        cParams = ZSTD_getCParams(1, ZSTD_FUZZ_GENERATED_SRC_MAXSIZE, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
+        cParams.minMatch = ZSTD_MINMATCH_MIN;
+        cParams.hashLog = ZSTD_HASHLOG_MIN;
+        cParams.chainLog = ZSTD_CHAINLOG_MIN;
+
+        cdict = ZSTD_createCDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
+        ddict = ZSTD_createDDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
+        FUZZ_ASSERT(cdict);
+        FUZZ_ASSERT(ddict);
+    }
+
+    FUZZ_ASSERT(cdict);
+    FUZZ_ASSERT(ddict);
+
     hasDict = FUZZ_dataProducer_uint32Range(producer, 0, 1);
     if (hasDict) {
-        dictSize = FUZZ_dataProducer_uint32Range(producer, 1, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
-        dictBuffer = FUZZ_malloc(dictSize);
-        FUZZ_ASSERT(dictBuffer);
-        dictBuffer = generatePseudoRandomString(dictBuffer, dictSize);
+        dictSize = ZSTD_FUZZ_GENERATED_DICT_MAXSIZE;
     }
-    /* Generate window log first so we don't generate offsets too large */
-    wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX_32);
-    cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
-    mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
 
     if (!generatedSequences) {
         generatedSequences = FUZZ_malloc(sizeof(ZSTD_Sequence)*ZSTD_FUZZ_MAX_NBSEQ);
@@ -305,8 +340,10 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
     if (!generatedSrc) {
         generatedSrc = FUZZ_malloc(ZSTD_FUZZ_GENERATED_SRC_MAXSIZE);
     }
+
     nbSequences = generateRandomSequences(producer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictSize, wLog, mode);
     generatedSrcSize = decodeSequences(generatedSrc, nbSequences, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictBuffer, dictSize, mode);
+
     /* Note : in explicit block delimiters mode,
      * the fuzzer might generate a lot of small blocks.
      * In which case, the final compressed size might be > ZSTD_compressBound().
@@ -318,30 +355,17 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
     rBufSize = generatedSrcSize;
     rBuf = FUZZ_malloc(rBufSize);
 
-    if (!cctx) {
-        cctx = ZSTD_createCCtx();
-        FUZZ_ASSERT(cctx);
-    }
-    if (!dctx) {
-        dctx = ZSTD_createDCtx();
-        FUZZ_ASSERT(dctx);
-    }
-
     {   const size_t result = roundTripTest(rBuf, rBufSize,
                                         cBuf, cBufSize,
                                         generatedSrc, generatedSrcSize,
-                                        dictBuffer, dictSize,
                                         generatedSequences, nbSequences,
-                                        (int)wLog, cLevel, hasDict, mode);
+                                        hasDict, mode);
         FUZZ_ASSERT(result <= generatedSrcSize);  /* can be 0 when no round-trip */
     }
 
     free(rBuf);
     free(cBuf);
     FUZZ_dataProducer_free(producer);
-    if (hasDict) {
-        free(dictBuffer);
-    }
 #ifndef STATEFUL_FUZZING
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;
diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c
index 8e6980b35..c9fea22d3 100644
--- a/tests/fuzz/simple_compress.c
+++ b/tests/fuzz/simple_compress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c
index dfff11cbc..ce5f9f098 100644
--- a/tests/fuzz/simple_decompress.c
+++ b/tests/fuzz/simple_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c
index 9da986bc9..c2c69d950 100644
--- a/tests/fuzz/simple_round_trip.c
+++ b/tests/fuzz/simple_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -26,6 +26,23 @@
 static ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
 
+static size_t getDecompressionMargin(void const* compressed, size_t cSize, size_t srcSize, int hasSmallBlocks)
+{
+    size_t margin = ZSTD_decompressionMargin(compressed, cSize);
+    if (!hasSmallBlocks) {
+        /* The macro should be correct in this case, but it may be smaller
+         * because of e.g. block splitting, so take the smaller of the two.
+         */
+        ZSTD_frameHeader zfh;
+        size_t marginM;
+        FUZZ_ZASSERT(ZSTD_getFrameHeader(&zfh, compressed, cSize));
+        marginM = ZSTD_DECOMPRESSION_MARGIN(srcSize, zfh.blockSizeMax);
+        if (marginM < margin)
+            margin = marginM;
+    }
+    return margin;
+}
+
 static size_t roundTripTest(void *result, size_t resultCapacity,
                             void *compressed, size_t compressedCapacity,
                             const void *src, size_t srcSize,
@@ -67,6 +84,25 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
     }
     dSize = ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
     FUZZ_ZASSERT(dSize);
+    FUZZ_ASSERT_MSG(dSize == srcSize, "Incorrect regenerated size");
+    FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, result, dSize), "Corruption!");
+
+    {
+        size_t margin = getDecompressionMargin(compressed, cSize, srcSize, targetCBlockSize);
+        size_t const outputSize = srcSize + margin;
+        char* const output = (char*)FUZZ_malloc(outputSize);
+        char* const input = output + outputSize - cSize;
+        FUZZ_ASSERT(outputSize >= cSize);
+        memcpy(input, compressed, cSize);
+
+        dSize = ZSTD_decompressDCtx(dctx, output, outputSize, input, cSize);
+        FUZZ_ZASSERT(dSize);
+        FUZZ_ASSERT_MSG(dSize == srcSize, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, output, srcSize), "Corruption!");
+
+        free(output);
+    }
+
     /* When superblock is enabled make sure we don't expand the block more than expected.
      * NOTE: This test is currently disabled because superblock mode can arbitrarily
      * expand the block in the worst case. Once superblock mode has been improved we can
@@ -120,13 +156,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
         FUZZ_ASSERT(dctx);
     }
 
-    {
-        size_t const result =
-            roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer);
-        FUZZ_ZASSERT(result);
-        FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
-        FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!");
-    }
+    roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer);
     free(rBuf);
     free(cBuf);
     FUZZ_dataProducer_free(producer);
diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c
index e0cdd34d9..0254d06ec 100644
--- a/tests/fuzz/stream_decompress.c
+++ b/tests/fuzz/stream_decompress.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -99,14 +99,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 
     while (size > 0) {
         ZSTD_inBuffer in = makeInBuffer(&src, &size, producer);
-        while (in.pos != in.size) {
+        do {
+            size_t const rc = ZSTD_decompressStream(dstream, &out, &in);
+            if (ZSTD_isError(rc)) goto error;
             if (out.pos == out.size) {
                 if (stableOutBuffer) goto error;
                 out = makeOutBuffer(producer, buf, bufSize);
             }
-            size_t const rc = ZSTD_decompressStream(dstream, &out, &in);
-            if (ZSTD_isError(rc)) goto error;
-        }
+        } while (in.pos != in.size);
     }
 
 error:
diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c
index 719eac48d..fae9ccbf4 100644
--- a/tests/fuzz/stream_round_trip.c
+++ b/tests/fuzz/stream_round_trip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -166,6 +166,24 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
         FUZZ_ZASSERT(rSize);
         FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size");
         FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!");
+
+        /* Test in-place decompression (note the macro doesn't work in this case) */
+        {
+            size_t const margin = ZSTD_decompressionMargin(cBuf, cSize);
+            size_t const outputSize = size + margin;
+            char* const output = (char*)FUZZ_malloc(outputSize);
+            char* const input = output + outputSize - cSize;
+            size_t dSize;
+            FUZZ_ASSERT(outputSize >= cSize);
+            memcpy(input, cBuf, cSize);
+
+            dSize = ZSTD_decompressDCtx(dctx, output, outputSize, input, cSize);
+            FUZZ_ZASSERT(dSize);
+            FUZZ_ASSERT_MSG(dSize == size, "Incorrect regenerated size");
+            FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, output, size), "Corruption!");
+
+            free(output);
+        }
     }
 
     FUZZ_dataProducer_free(producer);
diff --git a/tests/fuzz/zstd_frame_info.c b/tests/fuzz/zstd_frame_info.c
index 9ce645d15..95dbdd49a 100644
--- a/tests/fuzz/zstd_frame_info.c
+++ b/tests/fuzz/zstd_frame_info.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 08ce70dd7..6b833471e 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -17,6 +17,7 @@
 #include "fuzz_helpers.h"
 #include "zstd.h"
 #include "zdict.h"
+#include "matchfinder.h"
 
 const int kMinClevel = -3;
 const int kMaxClevel = 19;
@@ -26,9 +27,14 @@ static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value)
     FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
 }
 
+static unsigned produceParamValue(unsigned min, unsigned max,
+                                  FUZZ_dataProducer_t *producer) {
+    return FUZZ_dataProducer_uint32Range(producer, min, max);
+}
+
 static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min,
                     unsigned max, FUZZ_dataProducer_t *producer) {
-    unsigned const value = FUZZ_dataProducer_uint32Range(producer, min, max);
+    unsigned const value = produceParamValue(min, max, producer);
     set(cctx, param, value);
 }
 
@@ -65,6 +71,17 @@ ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer)
     return params;
 }
 
+static void setExternalMatchFinderParams(ZSTD_CCtx *cctx, FUZZ_dataProducer_t *producer) {
+    ZSTD_registerExternalMatchFinder(
+        cctx,
+        NULL,
+        simpleExternalMatchFinder
+    );
+    setRand(cctx, ZSTD_c_enableMatchFinderFallback, 0, 1, producer);
+    FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0));
+    FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
+}
+
 void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer)
 {
     ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, producer);
@@ -90,8 +107,14 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
             ZSTD_LDM_HASHRATELOG_MAX, producer);
     /* Set misc parameters */
 #ifndef ZSTD_MULTITHREAD
-    setRand(cctx, ZSTD_c_nbWorkers, 0, 0, producer);
-    setRand(cctx, ZSTD_c_rsyncable, 0, 0, producer);
+    // To reproduce with or without ZSTD_MULTITHREAD, we are going to use
+    // the same amount of entropy.
+    unsigned const nbWorkers_value = produceParamValue(0, 2, producer);
+    unsigned const rsyncable_value = produceParamValue(0, 1, producer);
+    (void)nbWorkers_value;
+    (void)rsyncable_value;
+    set(cctx, ZSTD_c_nbWorkers, 0);
+    set(cctx, ZSTD_c_rsyncable, 0);
 #else
     setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer);
     setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer);
@@ -104,12 +127,21 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer);
     setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer);
     setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer);
+    setRand(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN, ZSTD_BLOCKSIZE_MAX, producer);
+    setRand(cctx, ZSTD_c_validateSequences, 0, 1, producer);
+    setRand(cctx, ZSTD_c_searchForExternalRepcodes, 0, 2, producer);
     if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
       setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
     }
     if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
       setRand(cctx, ZSTD_c_targetCBlockSize, ZSTD_TARGETCBLOCKSIZE_MIN, ZSTD_TARGETCBLOCKSIZE_MAX, producer);
     }
+
+    if (FUZZ_dataProducer_uint32Range(producer, 0, 10) == 1) {
+        setExternalMatchFinderParams(cctx, producer);
+    } else {
+        ZSTD_registerExternalMatchFinder(cctx, NULL, NULL);
+    }
 }
 
 FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer)
diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h
index 9fbefdc72..a4cfe321b 100644
--- a/tests/fuzz/zstd_helpers.h
+++ b/tests/fuzz/zstd_helpers.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 879e537bc..fc78c7fbc 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -25,7 +25,8 @@
 #include        /* free */
 #include         /* fgets, sscanf */
 #include        /* strcmp */
-#undef NDEBUG
+#include          /* time(), time_t */
+#undef NDEBUG             /* always enable assert() */
 #include 
 #define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_compressContinue, ZSTD_compressBlock */
 #include "debug.h"        /* DEBUG_STATIC_ASSERT */
@@ -82,8 +83,8 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
 void FUZ_bug976(void);
 void FUZ_bug976(void)
 {   /* these constants shall not depend on MIN() macro */
-    assert(ZSTD_HASHLOG_MAX < 31);
-    assert(ZSTD_CHAINLOG_MAX < 31);
+    DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
+    DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
 }
 
 
@@ -118,23 +119,24 @@ static U32 FUZ_highbit32(U32 v32)
 /*=============================================
 *   Test macros
 =============================================*/
-#define CHECK_Z(f) {                               \
-    size_t const err = f;                          \
-    if (ZSTD_isError(err)) {                       \
-        DISPLAY("Error => %s : %s ",               \
-                #f, ZSTD_getErrorName(err));       \
-        exit(1);                                   \
+#define CHECK(fn)  { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }
+
+#define CHECK_Z(f) {                          \
+    size_t const err = f;                     \
+    if (ZSTD_isError(err)) {                  \
+        DISPLAY("Error => %s : %s ",          \
+                #f, ZSTD_getErrorName(err));  \
+        exit(1);                              \
 }   }
 
-#define CHECK_VAR(var, fn)  var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; }
+#define CHECK_VAR(var, fn)  var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
 #define CHECK_NEWV(var, fn)  size_t const CHECK_VAR(var, fn)
-#define CHECK(fn)  { CHECK_NEWV(__err, fn); }
 #define CHECKPLUS(var, fn, more)  { CHECK_NEWV(var, fn); more; }
 
 #define CHECK_OP(op, lhs, rhs) {                                  \
     if (!((lhs) op (rhs))) {                                      \
         DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs);  \
-        goto _output_error;                                       \
+         exit(1);                                                 \
     }                                                             \
 }
 #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
@@ -338,6 +340,7 @@ static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
 }
 
 #ifdef ZSTD_MULTITHREAD
+
 typedef struct {
     ZSTD_CCtx* cctx;
     ZSTD_threadPool* pool;
@@ -429,8 +432,8 @@ static int threadPoolTests(void) {
 
         ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
         ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
-        ZSTD_pthread_join(t1, NULL);
-        ZSTD_pthread_join(t2, NULL);
+        ZSTD_pthread_join(t1);
+        ZSTD_pthread_join(t2);
 
         assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
         free(decodedBuffer2);
@@ -461,6 +464,101 @@ _output_error:
 *   Unit tests
 =============================================*/
 
+static void test_compressBound(unsigned tnb)
+{
+    DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb);
+
+    /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
+     * for a large range of known valid values */
+    DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
+    {   int s;
+        for (s=0; s<30; s++) {
+            size_t const w = (size_t)1 << s;
+            CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
+    }   }
+
+    /* Ensure error if srcSize too big */
+    {   size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
+        CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
+        CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
+    }
+
+    DISPLAYLEVEL(3, "OK \n");
+}
+
+static void test_decompressBound(unsigned tnb)
+{
+    DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
+
+    /* Simple compression, with size : should provide size; */
+    {   const char example[] = "abcd";
+        char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
+        size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
+        CHECK_Z(cSize);
+        CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
+    }
+
+    /* Simple small compression without size : should provide 1 block size */
+    {   char cBuffer[ZSTD_COMPRESSBOUND(0)];
+        ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
+        ZSTD_inBuffer in = { NULL, 0, 0 };
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        assert(cctx);
+        CHECK_Z( ZSTD_initCStream(cctx, 0) );
+        CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
+        CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
+        CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX );
+        ZSTD_freeCCtx(cctx);
+    }
+
+    /* Attempt to overflow 32-bit intermediate multiplication result
+     * This requires dBound >= 4 GB, aka 2^32.
+     * This requires 2^32 / 2^17 = 2^15 blocks
+     * => create 2^15 blocks (can be empty, or just 1 byte). */
+    {   const char input[] = "a";
+        size_t const nbBlocks = (1 << 15) + 1;
+        size_t blockNb;
+        size_t const outCapacity = 1 << 18; /* large margin */
+        char* const outBuffer = malloc (outCapacity);
+        ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        assert(cctx);
+        assert(outBuffer);
+        CHECK_Z( ZSTD_initCStream(cctx, 0) );
+        for (blockNb=0; blockNb 0x100000000LLU /* 4 GB */ );
+        ZSTD_freeCCtx(cctx);
+        free(outBuffer);
+    }
+
+    DISPLAYLEVEL(3, "OK \n");
+}
+
+static void test_setCParams(unsigned tnb)
+{
+    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    ZSTD_compressionParameters cparams;
+    assert(cctx);
+
+    DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb);
+
+    /* valid cparams */
+    cparams = ZSTD_getCParams(1, 0, 0);
+    CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
+
+    /* invalid cparams (must fail) */
+    cparams.windowLog = 99;
+    CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
+
+    free(cctx);
+    DISPLAYLEVEL(3, "OK \n");
+}
+
 static int basicUnitTests(U32 const seed, double compressibility)
 {
     size_t const CNBuffSize = 5 MB;
@@ -507,6 +605,12 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "%u (OK) \n", vn);
     }
 
+    test_compressBound(testNb++);
+
+    test_decompressBound(testNb++);
+
+    test_setCParams(testNb++);
+
     DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
     {
         ZSTD_compressionParameters params;
@@ -580,6 +684,17 @@ static int basicUnitTests(U32 const seed, double compressibility)
       if (r != CNBuffSize) goto _output_error; }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize);
+    {
+        ZSTD_DCtx* dctx = ZSTD_createDCtx();
+        size_t r;
+        CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1));
+        r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+        if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+        ZSTD_freeDCtx(dctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
     {   size_t u;
         for (u=0; u %u bytes)\n", (unsigned)inputSize, (unsigned)result);
@@ -2856,7 +3218,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
         {   ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
-            CHECK(bounds.error);
+            CHECK_Z(bounds.error);
         }
         DISPLAYLEVEL(3, "OK \n");
 
@@ -2890,7 +3252,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* basic block compression */
         DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
         {   ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
             ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
             size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
@@ -2908,7 +3270,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
-        CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
         {   ZSTD_frameHeader zfh;
             size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
             if (zfhrt != 0) goto _output_error;
@@ -2930,7 +3292,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* basic block compression */
         DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
         {   ZSTD_inBuffer in = { CNBuffer, 0, 0 };
             ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
             size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
@@ -2942,7 +3304,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
-        CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
+        CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
         /* one shot */
         {   size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
             if (result != 0) goto _output_error;
@@ -2977,13 +3339,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
             int check;
             if (ZSTD_isError(bounds.error))
                 continue;
-            CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
+            CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
             value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
-            CHECK(ZSTD_DCtx_setParameter(dctx, dParam, value2));
-            CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
+            CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2));
+            CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
             if (check != value2) goto _output_error;
-            CHECK(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
-            CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
+            CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
+            CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
             if (check != value1) goto _output_error;
         }
         ZSTD_freeDCtx(dctx);
@@ -3000,21 +3362,21 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* basic block compression */
         DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
-        CHECK( ZSTD_compressBegin(cctx, 5) );
-        CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
+        CHECK_Z( ZSTD_compressBegin(cctx, 5) );
+        CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
         CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
-        CHECK( ZSTD_decompressBegin(dctx) );
+        CHECK_Z( ZSTD_decompressBegin(dctx) );
         { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
           if (r != blockSize) goto _output_error; }
         DISPLAYLEVEL(3, "OK \n");
 
         /* very long stream of block compression */
         DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
-        CHECK( ZSTD_compressBegin(cctx, -199) );  /* we just want to quickly overflow internal U32 index */
-        CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
+        CHECK_Z( ZSTD_compressBegin(cctx, -199) );  /* we just want to quickly overflow internal U32 index */
+        CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
         {   U64 const toCompress = 5000000000ULL;   /* > 4 GB */
             U64 compressed = 0;
             while (compressed < toCompress) {
@@ -3027,7 +3389,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* dictionary block compression */
         DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
-        CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
+        CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
         CHECK_VAR(cSize,  ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
         RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed);  /* create a non-compressible second block */
         { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) );  /* for cctx history consistency */
@@ -3038,7 +3400,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
-        CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
+        CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
         {   CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
             if (r != blockSize) {
                 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
@@ -3057,8 +3419,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
         {   ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
             if (cdict==NULL) goto _output_error;
-            CHECK( ZSTD_compressBegin_usingCDict(cctx, cdict) );
-            CHECK( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
+            CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) );
+            CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
             ZSTD_freeCDict(cdict);
         }
         DISPLAYLEVEL(3, "OK \n");
@@ -3227,7 +3589,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
         size_t nbSeq = 1;
         while (nbSeq <= maxNbSeq) {
-          CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
+          CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
           /* Check every sequence for the first 100, then skip more rapidly. */
           if (nbSeq < 100) {
             ++nbSeq;
@@ -3256,7 +3618,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         size_t const bound = ZSTD_compressBound(CNBuffSize);
         size_t size = 1;
         while (size <= CNBuffSize) {
-          CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
+          CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
           /* Check every size for the first 100, then skip more rapidly. */
           if (size < 100) {
             ++size;
@@ -3291,7 +3653,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             void* const outputBuffer = malloc(outputSize);
             ZSTD_CCtx* const cctx = ZSTD_createCCtx();
             if (!outputBuffer || !cctx) goto _output_error;
-            CHECK(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
+            CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
             free(outputBuffer);
             ZSTD_freeCCtx(cctx);
         }
@@ -3378,7 +3740,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
     {
-        U32 seed_copy = seed; // need non-const seed to avoid compiler warning for FUZ_rand(&seed)
+        U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
         U32 rand32 = FUZ_rand(&seed_copy);
         U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
         U32 lowbit_only_32 = 1;
@@ -3386,8 +3748,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
         U32 highbit_only_32 = (U32)1 << 31;
         U64 highbit_only_64 = (U64)1 << 63;
         U32 i;
-        if (rand32 == 0) rand32 = 1; // CLZ and CTZ are undefined on 0
-        if (rand64 == 0) rand64 = 1; // CLZ and CTZ are undefined on 0
+        if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
+        if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
 
         /* Test ZSTD_countTrailingZeros32 */
         CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);
@@ -3628,7 +3990,7 @@ static int longUnitTests(U32 const seed, double compressibility)
             while (approxIndex <= (maxIndex / 4) * 3) {
                 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
                 approxIndex += in.pos;
-                CHECK(in.pos == in.size);
+                CHECK_Z(in.pos == in.size);
                 in.pos = 0;
                 out.pos = 0;
             }
@@ -3654,7 +4016,7 @@ static int longUnitTests(U32 const seed, double compressibility)
             while (approxIndex <= maxIndex) {
                 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
                 approxIndex += in.pos;
-                CHECK(in.pos == in.size);
+                CHECK_Z(in.pos == in.size);
                 in.pos = 0;
                 out.pos = 0;
             }
@@ -3737,7 +4099,7 @@ static int longUnitTests(U32 const seed, double compressibility)
         RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
         RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
 
-        CHECK(cctx_params != NULL);
+        CHECK_Z(cctx_params != NULL);
 
         for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
             DISPLAYLEVEL(3, "\n    Testing with dictSize %u ", (U32)dictSize);
@@ -3780,11 +4142,6 @@ _end:
     free(compressedBuffer);
     free(decodedBuffer);
     return testResult;
-
-_output_error:
-    testResult = 1;
-    DISPLAY("Error detected in Unit tests ! \n");
-    goto _end;
 }
 
 
diff --git a/tests/golden-decompression/empty-block.zst b/tests/golden-decompression/empty-block.zst
index 2a3782aff0d1a6bab62dcdc1af29b448322a84bd..fbfb893e11eb677f1e6444ead8a5829a3a23e53e 100644
GIT binary patch
literal 11
QcmdPcs{faPL6iXq020jt9{>OV

literal 11
QcmdPcs{dDkL6iXq028eOKL7v#

diff --git a/tests/gzip/Makefile b/tests/gzip/Makefile
index a50350f8d..cca310967 100644
--- a/tests/gzip/Makefile
+++ b/tests/gzip/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/gzip/gzip-env.sh b/tests/gzip/gzip-env.sh
index 120e52d78..4570e81c3 100755
--- a/tests/gzip/gzip-env.sh
+++ b/tests/gzip/gzip-env.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/helin-segv.sh b/tests/gzip/helin-segv.sh
index f182c8066..b400c2445 100644
--- a/tests/gzip/helin-segv.sh
+++ b/tests/gzip/helin-segv.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/help-version.sh b/tests/gzip/help-version.sh
index ee0c19f7d..fcda1c3ac 100644
--- a/tests/gzip/help-version.sh
+++ b/tests/gzip/help-version.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 
 # Ensure that $SHELL is set to *some* value and exported.
 # This is required for dircolors, which would fail e.g., when
diff --git a/tests/gzip/hufts.sh b/tests/gzip/hufts.sh
index 9b9576ce3..49c369501 100644
--- a/tests/gzip/hufts.sh
+++ b/tests/gzip/hufts.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/init.sh b/tests/gzip/init.sh
index 97e4e4ba5..ebd8410be 100644
--- a/tests/gzip/init.sh
+++ b/tests/gzip/init.sh
@@ -13,7 +13,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 
 # Using this file in a test
 # =========================
diff --git a/tests/gzip/keep.sh b/tests/gzip/keep.sh
index ab9a21811..f87b1a49b 100644
--- a/tests/gzip/keep.sh
+++ b/tests/gzip/keep.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/list.sh b/tests/gzip/list.sh
index 75912e1e2..e218d7562 100644
--- a/tests/gzip/list.sh
+++ b/tests/gzip/list.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/memcpy-abuse.sh b/tests/gzip/memcpy-abuse.sh
index 7d5c056de..1478890cc 100644
--- a/tests/gzip/memcpy-abuse.sh
+++ b/tests/gzip/memcpy-abuse.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/mixed.sh b/tests/gzip/mixed.sh
index 383a54f5e..b47f4a56a 100644
--- a/tests/gzip/mixed.sh
+++ b/tests/gzip/mixed.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/null-suffix-clobber.sh b/tests/gzip/null-suffix-clobber.sh
index 0efd0e344..5acfb321c 100644
--- a/tests/gzip/null-suffix-clobber.sh
+++ b/tests/gzip/null-suffix-clobber.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/stdin.sh b/tests/gzip/stdin.sh
index eef4cd8b1..d697ab80a 100644
--- a/tests/gzip/stdin.sh
+++ b/tests/gzip/stdin.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/test-driver.sh b/tests/gzip/test-driver.sh
index 649c084e4..0529cc8fb 100644
--- a/tests/gzip/test-driver.sh
+++ b/tests/gzip/test-driver.sh
@@ -16,7 +16,7 @@ scriptversion=2016-01-11.22; # UTC
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 
 # As a special exception to the GNU General Public License, if you
 # distribute this file as part of a program that contains a
diff --git a/tests/gzip/trailing-nul.sh b/tests/gzip/trailing-nul.sh
index 7b15d5e55..b33b98f78 100644
--- a/tests/gzip/trailing-nul.sh
+++ b/tests/gzip/trailing-nul.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/unpack-invalid.sh b/tests/gzip/unpack-invalid.sh
index fe8384d73..ceda5ffa4 100644
--- a/tests/gzip/unpack-invalid.sh
+++ b/tests/gzip/unpack-invalid.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/z-suffix.sh b/tests/gzip/z-suffix.sh
index a870a5408..c0bf509c9 100644
--- a/tests/gzip/z-suffix.sh
+++ b/tests/gzip/z-suffix.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/zdiff.sh b/tests/gzip/zdiff.sh
index d62a84606..6e99b66f4 100644
--- a/tests/gzip/zdiff.sh
+++ b/tests/gzip/zdiff.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/zgrep-context.sh b/tests/gzip/zgrep-context.sh
index c8648b7e4..d213426a3 100644
--- a/tests/gzip/zgrep-context.sh
+++ b/tests/gzip/zgrep-context.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/zgrep-f.sh b/tests/gzip/zgrep-f.sh
index d0cf27f7e..1e73ed204 100644
--- a/tests/gzip/zgrep-f.sh
+++ b/tests/gzip/zgrep-f.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/zgrep-signal.sh b/tests/gzip/zgrep-signal.sh
index a8c53881a..dd8442c81 100644
--- a/tests/gzip/zgrep-signal.sh
+++ b/tests/gzip/zgrep-signal.sh
@@ -15,7 +15,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/gzip/znew-k.sh b/tests/gzip/znew-k.sh
index 6c239e28e..5cf99ed34 100644
--- a/tests/gzip/znew-k.sh
+++ b/tests/gzip/znew-k.sh
@@ -14,7 +14,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
+# along with this program.  If not, see .
 # limit so don't run it by default.
 
 . "${srcdir=.}/init.sh"; path_prepend_ .
diff --git a/tests/invalidDictionaries.c b/tests/invalidDictionaries.c
index b71f7419f..66caa9e99 100644
--- a/tests/invalidDictionaries.c
+++ b/tests/invalidDictionaries.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/legacy.c b/tests/legacy.c
index ac4938fe5..3be386495 100644
--- a/tests/legacy.c
+++ b/tests/legacy.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/longmatch.c b/tests/longmatch.c
index a171c0e4a..547b26190 100644
--- a/tests/longmatch.c
+++ b/tests/longmatch.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -36,27 +36,27 @@ compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size)
 int main(int argc, const char** argv)
 {
   ZSTD_CStream* ctx;
-  ZSTD_parameters params;
-  size_t rc;
-  unsigned windowLog;
+  unsigned windowLog = 18;
   (void)argc;
   (void)argv;
   /* Create stream */
-  ctx = ZSTD_createCStream();
+  ctx = ZSTD_createCCtx();
   if (!ctx) { return 1; }
   /* Set parameters */
-  memset(¶ms, 0, sizeof(params));
-  params.cParams.windowLog = 18;
-  params.cParams.chainLog = 13;
-  params.cParams.hashLog = 14;
-  params.cParams.searchLog = 1;
-  params.cParams.minMatch = 7;
-  params.cParams.targetLength = 16;
-  params.cParams.strategy = ZSTD_fast;
-  windowLog = params.cParams.windowLog;
-  /* Initialize stream */
-  rc = ZSTD_initCStream_advanced(ctx, NULL, 0, params, 0);
-  if (ZSTD_isError(rc)) { return 2; }
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, windowLog)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, 13)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, 14)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, 1)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, 7)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, 16)))
+    return 2;
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, ZSTD_fast)))
+    return 2;
   {
     U64 compressed = 0;
     const U64 toCompress = ((U64)1) << 33;
@@ -97,5 +97,6 @@ int main(int argc, const char** argv)
     free(srcBuffer);
     free(dstBuffer);
   }
+  ZSTD_freeCCtx(ctx);
   return 0;
 }
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 756a43340..8971c65d6 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -566,10 +566,10 @@ resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_
     double cs = 0., ds = 0., rt, cm = 0.;
     const double r1 = 1, r2 = 0.1, rtr = 0.5;
     double ret;
-    if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; }
-    if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; }
-    if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; }
-    rt = ((double)srcSize / res.cSize);
+    if(target.cSpeed) { cs = (double)res.cSpeed / (double)target.cSpeed; }
+    if(target.dSpeed) { ds = (double)res.dSpeed / (double)target.dSpeed; }
+    if(target.cMem != (U32)-1) { cm = (double)target.cMem / (double)res.cMem; }
+    rt = ((double)srcSize / (double)res.cSize);
 
     ret = (MIN(1, cs) + MIN(1, ds)  + MIN(1, cm))*r1 + rt * rtr +
          (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2;
@@ -581,8 +581,8 @@ resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_
 static double
 resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResult_t lvlRes)
 {
-    double normalizedCSpeedGain1 = ((double)result1.cSpeed / lvlRes.cSpeed) - 1;
-    double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
+    double normalizedCSpeedGain1 = ((double)result1.cSpeed / (double)lvlRes.cSpeed) - 1;
+    double normalizedRatioGain1 = ((double)lvlRes.cSize / (double)result1.cSize) - 1;
     if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
         return 0.0;
     }
@@ -854,7 +854,7 @@ BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize)
     }
 
     {   double const ratio = res.result.cSize ?
-                            (double)srcSize / res.result.cSize : 0;
+                            (double)srcSize / (double)res.result.cSize : 0;
         double const cSpeedMBps = (double)res.result.cSpeed / MB_UNIT;
         double const dSpeedMBps = (double)res.result.dSpeed / MB_UNIT;
 
@@ -937,7 +937,7 @@ BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_benchResult_t result, co
         }
         fprintf(f, "================================\n");
         fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
-            (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / MB_UNIT);
+            (double)srcSize / (double)g_lvltarget.cSize, (double)g_lvltarget.cSpeed / MB_UNIT);
 
 
         fprintf(f, "Overall Winner: \n");
@@ -977,7 +977,7 @@ BMK_print_cLevelEntry(FILE* f, const int cLevel,
     }
     /* print comment */
     {   double const ratio = result.cSize ?
-                            (double)srcSize / result.cSize : 0;
+                            (double)srcSize / (double)result.cSize : 0;
         double const cSpeedMBps = (double)result.cSpeed / MB_UNIT;
         double const dSpeedMBps = (double)result.dSpeed / MB_UNIT;
 
@@ -1726,19 +1726,19 @@ static int allBench(BMK_benchResult_t* resultPtr,
 
     /* calculate uncertainty in compression / decompression runs */
     if (benchres.cSpeed) {
-        U64 const loopDurationC = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
+        double const loopDurationC = (double)(((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
         uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC);
     }
 
     if (benchres.dSpeed) {
-        U64 const loopDurationD = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
+        double const loopDurationD = (double)(((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
         uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);
     }
 
     /* optimistic assumption of benchres */
     {   BMK_benchResult_t resultMax = benchres;
-        resultMax.cSpeed = (unsigned long long)(resultMax.cSpeed * uncertaintyConstantC * VARIANCE);
-        resultMax.dSpeed = (unsigned long long)(resultMax.dSpeed * uncertaintyConstantD * VARIANCE);
+        resultMax.cSpeed = (unsigned long long)((double)resultMax.cSpeed * uncertaintyConstantC * VARIANCE);
+        resultMax.dSpeed = (unsigned long long)((double)resultMax.dSpeed * uncertaintyConstantD * VARIANCE);
 
         /* disregard infeasible results in feas mode */
         /* disregard if resultMax < winner in infeas mode */
@@ -1850,8 +1850,8 @@ static int BMK_seed(winnerInfo_t* winners,
 
         if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
             /* Validate solution is "good enough" */
-            double W_ratio = (double)buf.srcSize / testResult.cSize;
-            double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize;
+            double W_ratio = (double)buf.srcSize / (double)testResult.cSize;
+            double O_ratio = (double)buf.srcSize / (double)winners[cLevel].result.cSize;
             double W_ratioNote = log (W_ratio);
             double O_ratioNote = log (O_ratio);
             size_t W_DMemUsed = (1 << params.vals[wlog_ind]) + (16 KB);
@@ -1864,11 +1864,11 @@ static int BMK_seed(winnerInfo_t* winners,
             double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
             double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
 
-            double W_CSpeed_note = W_ratioNote * (double)( 30 + 10*cLevel) + log(testResult.cSpeed);
-            double O_CSpeed_note = O_ratioNote * (double)( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed);
+            double W_CSpeed_note = W_ratioNote * (double)( 30 + 10*cLevel) + log((double)testResult.cSpeed);
+            double O_CSpeed_note = O_ratioNote * (double)( 30 + 10*cLevel) + log((double)winners[cLevel].result.cSpeed);
 
-            double W_DSpeed_note = W_ratioNote * (double)( 20 + 2*cLevel) + log(testResult.dSpeed);
-            double O_DSpeed_note = O_ratioNote * (double)( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed);
+            double W_DSpeed_note = W_ratioNote * (double)( 20 + 2*cLevel) + log((double)testResult.dSpeed);
+            double O_DSpeed_note = O_ratioNote * (double)( 20 + 2*cLevel) + log((double)winners[cLevel].result.dSpeed);
 
             if (W_DMemUsed_note < O_DMemUsed_note) {
                 /* uses too much Decompression memory for too little benefit */
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 7c111e7a3..5f595f611 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -387,7 +387,29 @@ println "\n===>  file removal"
 zstd -f --rm tmp
 test ! -f tmp  # tmp should no longer be present
 zstd -f -d --rm tmp.zst
-test ! -f tmp.zst   # tmp.zst should no longer be present
+test ! -f tmp.zst  # tmp.zst should no longer be present
+println "test: --rm is disabled when output is stdout"
+test -f tmp
+zstd --rm tmp -c > $INTOVOID
+test -f tmp # tmp shall still be there
+zstd -f --rm tmp -c > $INTOVOID
+test -f tmp # tmp shall still be there
+zstd -f tmp -c > $INTOVOID --rm
+test -f tmp # tmp shall still be there
+println "test: --rm is disabled when multiple inputs are concatenated into a single output"
+cp tmp tmp2
+zstd --rm tmp tmp2 -c > $INTOVOID
+test -f tmp
+test -f tmp2
+rm -f tmp3.zst
+echo 'y' | zstd -v tmp tmp2 -o tmp3.zst --rm # prompt for confirmation
+test -f tmp
+test -f tmp2
+zstd -f tmp tmp2 -o tmp3.zst --rm # just warns, no prompt
+test -f tmp
+test -f tmp2
+zstd -q tmp tmp2 -o tmp3.zst --rm && die "should refuse to concatenate"
+
 println "test : should quietly not remove non-regular file"
 println hello > tmp
 zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID"
@@ -429,36 +451,35 @@ println hello > tmp1
 println world > tmp2
 zstd tmp1 tmp2 -o "$INTOVOID" -f
 zstd tmp1 tmp2 -c | zstd -t
-zstd tmp1 tmp2 -o tmp.zst
+echo 'y' | zstd -v tmp1 tmp2 -o tmp.zst
 test ! -f tmp1.zst
 test ! -f tmp2.zst
 zstd tmp1 tmp2
 zstd -t tmp1.zst tmp2.zst
 zstd -dc tmp1.zst tmp2.zst
 zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f
-zstd -d tmp1.zst tmp2.zst -o tmp
+echo 'y' | zstd -v -d tmp1.zst tmp2.zst -o tmp
 touch tmpexists
 zstd tmp1 tmp2 -f -o tmpexists
 zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite"
 println gooder > tmp_rm1
 println boi > tmp_rm2
 println worldly > tmp_rm3
-echo 'y' | zstd tmp_rm1 tmp_rm2 -v -o tmp_rm3.zst --rm     # tests the warning prompt for --rm with multiple inputs into once source
-test ! -f tmp_rm1
-test ! -f tmp_rm2
+echo 'y' | zstd -v tmp_rm1 tmp_rm2 -v -o tmp_rm3.zst
+test -f tmp_rm1
+test -f tmp_rm2
 cp tmp_rm3.zst tmp_rm4.zst
-echo 'Y' | zstd -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm
-test ! -f tmp_rm3.zst
-test ! -f tmp_rm4.zst
-echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm && die "compressing multiple files to stdout with --rm should fail unless -f is specified"
-echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm -v && die "compressing multiple files to stdout with --rm should fail unless -f is specified"
+echo 'Y' | zstd -v -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm
+test -f tmp_rm3.zst
+test -f tmp_rm4.zst
 println gooder > tmpexists1
 zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID
-
 # Bug: PR #972
 if [ "$?" -eq 139 ]; then
   die "should not have segfaulted"
 fi
+test -f tmpexists1
+test -f tmpexists
 println "\n===>  multiple files and shell completion "
 datagen -s1        > tmp1 2> $INTOVOID
 datagen -s2 -g100K > tmp2 2> $INTOVOID
@@ -562,19 +583,9 @@ if [ "$isWindows" = false ] ; then
     zstd -f -d tmp1.zst -o tmp1.out
     assertFilePermissions tmp1.out 400
 
-    rm -f tmp1.zst tmp1.out
-
     umask 0666
     chmod 0666 tmp1 tmp2
 
-    println "test : respect umask when copying permissions in file -> file compression "
-    zstd -f tmp1 -o tmp1.zst
-    assertFilePermissions tmp1.zst 0
-    println "test : respect umask when copying permissions in file -> file decompression "
-    chmod 0666 tmp1.zst
-    zstd -f -d tmp1.zst -o tmp1.out
-    assertFilePermissions tmp1.out 0
-
     rm -f tmp1.zst tmp1.out
 
     println "test : respect umask when compressing from stdin input "
@@ -1176,6 +1187,10 @@ zstd -t tmp3 && die "bad file not detected !"   # detects 0-sized files as bad
 println "test --rm and --test combined "
 zstd -t --rm tmp1.zst
 test -f tmp1.zst   # check file is still present
+cp tmp1.zst tmp2.zst
+zstd -t tmp1.zst tmp2.zst --rm
+test -f tmp1.zst   # check file is still present
+test -f tmp2.zst   # check file is still present
 split -b16384 tmp1.zst tmpSplit.
 zstd -t tmpSplit.* && die "bad file not detected !"
 datagen | zstd -c | zstd -t
@@ -1202,7 +1217,15 @@ zstd -rqi0b1e2 tmp1
 println "benchmark decompression only"
 zstd -f tmp1
 zstd -b -d -i0 tmp1.zst
+println "benchmark can fail - decompression on invalid data"
+zstd -b -d -i0 tmp1 && die "invalid .zst data => benchmark should have failed"
 
+GZIPMODE=1
+zstd --format=gzip -V || GZIPMODE=0
+if [ $GZIPMODE -eq 1 ]; then
+    println "benchmark mode is only compatible with zstd"
+    zstd --format=gzip -b tmp1 && die "-b should be incompatible with gzip format!"
+fi
 
 println "\n===>  zstd compatibility tests "
 
diff --git a/tests/poolTests.c b/tests/poolTests.c
index 08f31c069..9e62722bf 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/rateLimiter.py b/tests/rateLimiter.py
index 26293728e..3e65081e0 100755
--- a/tests/rateLimiter.py
+++ b/tests/rateLimiter.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/Makefile b/tests/regression/Makefile
index a440c6c94..ba8b43024 100644
--- a/tests/regression/Makefile
+++ b/tests/regression/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/config.c b/tests/regression/config.c
index 30d0ca5e2..1d86fff23 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/config.h b/tests/regression/config.h
index dd8893743..a4b542a90 100644
--- a/tests/regression/config.h
+++ b/tests/regression/config.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/data.c b/tests/regression/data.c
index 341b02d3b..43f085f2c 100644
--- a/tests/regression/data.c
+++ b/tests/regression/data.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/data.h b/tests/regression/data.h
index e54e6a1dc..a4ee92072 100644
--- a/tests/regression/data.h
+++ b/tests/regression/data.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/levels.h b/tests/regression/levels.h
index e98209d80..d15b12046 100644
--- a/tests/regression/levels.h
+++ b/tests/regression/levels.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/method.c b/tests/regression/method.c
index 55b1154eb..f84a15ef3 100644
--- a/tests/regression/method.c
+++ b/tests/regression/method.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/method.h b/tests/regression/method.h
index 1a36a9399..8efdd33a0 100644
--- a/tests/regression/method.h
+++ b/tests/regression/method.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/result.c b/tests/regression/result.c
index 1f879c140..8ccb8751e 100644
--- a/tests/regression/result.c
+++ b/tests/regression/result.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/result.h b/tests/regression/result.h
index 197fa905d..8a761ea4d 100644
--- a/tests/regression/result.h
+++ b/tests/regression/result.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index dee7d5721..188eea900 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -11,10 +11,10 @@ silesia.tar,                        level 6,                            compress
 silesia.tar,                        level 7,                            compress simple,                    4576661
 silesia.tar,                        level 9,                            compress simple,                    4552899
 silesia.tar,                        level 13,                           compress simple,                    4502956
-silesia.tar,                        level 16,                           compress simple,                    4360527
-silesia.tar,                        level 19,                           compress simple,                    4266970
+silesia.tar,                        level 16,                           compress simple,                    4360546
+silesia.tar,                        level 19,                           compress simple,                    4265911
 silesia.tar,                        uncompressed literals,              compress simple,                    4854086
-silesia.tar,                        uncompressed literals optimal,      compress simple,                    4266970
+silesia.tar,                        uncompressed literals optimal,      compress simple,                    4265911
 silesia.tar,                        huffman literals,                   compress simple,                    6179047
 github.tar,                         level -5,                           compress simple,                    52115
 github.tar,                         level -3,                           compress simple,                    45678
@@ -28,10 +28,10 @@ github.tar,                         level 6,                            compress
 github.tar,                         level 7,                            compress simple,                    38110
 github.tar,                         level 9,                            compress simple,                    36760
 github.tar,                         level 13,                           compress simple,                    35501
-github.tar,                         level 16,                           compress simple,                    40471
-github.tar,                         level 19,                           compress simple,                    32149
+github.tar,                         level 16,                           compress simple,                    40466
+github.tar,                         level 19,                           compress simple,                    32276
 github.tar,                         uncompressed literals,              compress simple,                    38831
-github.tar,                         uncompressed literals optimal,      compress simple,                    32149
+github.tar,                         uncompressed literals optimal,      compress simple,                    32276
 github.tar,                         huffman literals,                   compress simple,                    42560
 silesia,                            level -5,                           compress cctx,                      6857372
 silesia,                            level -3,                           compress cctx,                      6503412
@@ -45,8 +45,8 @@ silesia,                            level 6,                            compress
 silesia,                            level 7,                            compress cctx,                      4566984
 silesia,                            level 9,                            compress cctx,                      4543018
 silesia,                            level 13,                           compress cctx,                      4493990
-silesia,                            level 16,                           compress cctx,                      4359864
-silesia,                            level 19,                           compress cctx,                      4296438
+silesia,                            level 16,                           compress cctx,                      4360041
+silesia,                            level 19,                           compress cctx,                      4296055
 silesia,                            long distance mode,                 compress cctx,                      4842075
 silesia,                            multithreaded,                      compress cctx,                      4842075
 silesia,                            multithreaded long distance mode,   compress cctx,                      4842075
@@ -55,7 +55,7 @@ silesia,                            small hash log,                     compress
 silesia,                            small chain log,                    compress cctx,                      4912197
 silesia,                            explicit params,                    compress cctx,                      4794052
 silesia,                            uncompressed literals,              compress cctx,                      4842075
-silesia,                            uncompressed literals optimal,      compress cctx,                      4296438
+silesia,                            uncompressed literals optimal,      compress cctx,                      4296055
 silesia,                            huffman literals,                   compress cctx,                      6172202
 silesia,                            multithreaded with advanced params, compress cctx,                      4842075
 github,                             level -5,                           compress cctx,                      204407
@@ -80,11 +80,11 @@ github,                             level 7,                            compress
 github,                             level 7 with dict,                  compress cctx,                      38755
 github,                             level 9,                            compress cctx,                      135122
 github,                             level 9 with dict,                  compress cctx,                      39398
-github,                             level 13,                           compress cctx,                      132729
+github,                             level 13,                           compress cctx,                      132878
 github,                             level 13 with dict,                 compress cctx,                      39948
-github,                             level 16,                           compress cctx,                      132729
+github,                             level 16,                           compress cctx,                      133209
 github,                             level 16 with dict,                 compress cctx,                      37568
-github,                             level 19,                           compress cctx,                      132729
+github,                             level 19,                           compress cctx,                      132879
 github,                             level 19 with dict,                 compress cctx,                      37567
 github,                             long distance mode,                 compress cctx,                      141069
 github,                             multithreaded,                      compress cctx,                      141069
@@ -94,7 +94,7 @@ github,                             small hash log,                     compress
 github,                             small chain log,                    compress cctx,                      139242
 github,                             explicit params,                    compress cctx,                      140932
 github,                             uncompressed literals,              compress cctx,                      136332
-github,                             uncompressed literals optimal,      compress cctx,                      132729
+github,                             uncompressed literals optimal,      compress cctx,                      132879
 github,                             huffman literals,                   compress cctx,                      175468
 github,                             multithreaded with advanced params, compress cctx,                      141069
 silesia,                            level -5,                           zstdcli,                            6857420
@@ -109,8 +109,8 @@ silesia,                            level 6,                            zstdcli,
 silesia,                            level 7,                            zstdcli,                            4567032
 silesia,                            level 9,                            zstdcli,                            4543066
 silesia,                            level 13,                           zstdcli,                            4494038
-silesia,                            level 16,                           zstdcli,                            4359912
-silesia,                            level 19,                           zstdcli,                            4296486
+silesia,                            level 16,                           zstdcli,                            4360089
+silesia,                            level 19,                           zstdcli,                            4296103
 silesia,                            long distance mode,                 zstdcli,                            4833785
 silesia,                            multithreaded,                      zstdcli,                            4842123
 silesia,                            multithreaded long distance mode,   zstdcli,                            4833785
@@ -134,14 +134,14 @@ silesia.tar,                        level 6,                            zstdcli,
 silesia.tar,                        level 7,                            zstdcli,                            4578719
 silesia.tar,                        level 9,                            zstdcli,                            4552903
 silesia.tar,                        level 13,                           zstdcli,                            4502960
-silesia.tar,                        level 16,                           zstdcli,                            4360531
-silesia.tar,                        level 19,                           zstdcli,                            4266974
+silesia.tar,                        level 16,                           zstdcli,                            4360550
+silesia.tar,                        level 19,                           zstdcli,                            4265915
 silesia.tar,                        no source size,                     zstdcli,                            4854160
 silesia.tar,                        long distance mode,                 zstdcli,                            4845745
 silesia.tar,                        multithreaded,                      zstdcli,                            4854164
 silesia.tar,                        multithreaded long distance mode,   zstdcli,                            4845745
 silesia.tar,                        small window log,                   zstdcli,                            7100701
-silesia.tar,                        small hash log,                     zstdcli,                            6529289
+silesia.tar,                        small hash log,                     zstdcli,                            6529264
 silesia.tar,                        small chain log,                    zstdcli,                            4917022
 silesia.tar,                        explicit params,                    zstdcli,                            4820713
 silesia.tar,                        uncompressed literals,              zstdcli,                            5122571
@@ -170,11 +170,11 @@ github,                             level 7,                            zstdcli,
 github,                             level 7 with dict,                  zstdcli,                            40745
 github,                             level 9,                            zstdcli,                            137122
 github,                             level 9 with dict,                  zstdcli,                            41393
-github,                             level 13,                           zstdcli,                            134729
+github,                             level 13,                           zstdcli,                            134878
 github,                             level 13 with dict,                 zstdcli,                            41900
-github,                             level 16,                           zstdcli,                            134729
+github,                             level 16,                           zstdcli,                            135209
 github,                             level 16 with dict,                 zstdcli,                            39577
-github,                             level 19,                           zstdcli,                            134729
+github,                             level 19,                           zstdcli,                            134879
 github,                             level 19 with dict,                 zstdcli,                            39576
 github,                             long distance mode,                 zstdcli,                            138332
 github,                             multithreaded,                      zstdcli,                            138332
@@ -184,7 +184,7 @@ github,                             small hash log,                     zstdcli,
 github,                             small chain log,                    zstdcli,                            138341
 github,                             explicit params,                    zstdcli,                            136197
 github,                             uncompressed literals,              zstdcli,                            167911
-github,                             uncompressed literals optimal,      zstdcli,                            159227
+github,                             uncompressed literals optimal,      zstdcli,                            154667
 github,                             huffman literals,                   zstdcli,                            144365
 github,                             multithreaded with advanced params, zstdcli,                            167911
 github.tar,                         level -5,                           zstdcli,                            52119
@@ -211,9 +211,9 @@ github.tar,                         level 9,                            zstdcli,
 github.tar,                         level 9 with dict,                  zstdcli,                            36632
 github.tar,                         level 13,                           zstdcli,                            35505
 github.tar,                         level 13 with dict,                 zstdcli,                            37134
-github.tar,                         level 16,                           zstdcli,                            40475
+github.tar,                         level 16,                           zstdcli,                            40470
 github.tar,                         level 16 with dict,                 zstdcli,                            33378
-github.tar,                         level 19,                           zstdcli,                            32153
+github.tar,                         level 19,                           zstdcli,                            32280
 github.tar,                         level 19 with dict,                 zstdcli,                            32716
 github.tar,                         no source size,                     zstdcli,                            38832
 github.tar,                         no source size with dict,           zstdcli,                            38004
@@ -248,8 +248,8 @@ silesia,                            level 11 row 2,                     advanced
 silesia,                            level 12 row 1,                     advanced one pass,                  4505046
 silesia,                            level 12 row 2,                     advanced one pass,                  4503116
 silesia,                            level 13,                           advanced one pass,                  4493990
-silesia,                            level 16,                           advanced one pass,                  4359864
-silesia,                            level 19,                           advanced one pass,                  4296438
+silesia,                            level 16,                           advanced one pass,                  4360041
+silesia,                            level 19,                           advanced one pass,                  4296055
 silesia,                            no source size,                     advanced one pass,                  4842075
 silesia,                            long distance mode,                 advanced one pass,                  4833710
 silesia,                            multithreaded,                      advanced one pass,                  4842075
@@ -282,14 +282,14 @@ silesia.tar,                        level 11 row 2,                     advanced
 silesia.tar,                        level 12 row 1,                     advanced one pass,                  4514049
 silesia.tar,                        level 12 row 2,                     advanced one pass,                  4513797
 silesia.tar,                        level 13,                           advanced one pass,                  4502956
-silesia.tar,                        level 16,                           advanced one pass,                  4360527
-silesia.tar,                        level 19,                           advanced one pass,                  4266970
+silesia.tar,                        level 16,                           advanced one pass,                  4360546
+silesia.tar,                        level 19,                           advanced one pass,                  4265911
 silesia.tar,                        no source size,                     advanced one pass,                  4854086
 silesia.tar,                        long distance mode,                 advanced one pass,                  4840452
 silesia.tar,                        multithreaded,                      advanced one pass,                  4854160
 silesia.tar,                        multithreaded long distance mode,   advanced one pass,                  4845741
 silesia.tar,                        small window log,                   advanced one pass,                  7100655
-silesia.tar,                        small hash log,                     advanced one pass,                  6529231
+silesia.tar,                        small hash log,                     advanced one pass,                  6529206
 silesia.tar,                        small chain log,                    advanced one pass,                  4917041
 silesia.tar,                        explicit params,                    advanced one pass,                  4806855
 silesia.tar,                        uncompressed literals,              advanced one pass,                  5122473
@@ -370,39 +370,39 @@ github,                             level 9 with dict dms,              advanced
 github,                             level 9 with dict dds,              advanced one pass,                  39393
 github,                             level 9 with dict copy,             advanced one pass,                  39398
 github,                             level 9 with dict load,             advanced one pass,                  41710
-github,                             level 11 row 1,                     advanced one pass,                  135119
+github,                             level 11 row 1,                     advanced one pass,                  135367
 github,                             level 11 row 1 with dict dms,       advanced one pass,                  39671
 github,                             level 11 row 1 with dict dds,       advanced one pass,                  39671
 github,                             level 11 row 1 with dict copy,      advanced one pass,                  39651
 github,                             level 11 row 1 with dict load,      advanced one pass,                  41360
-github,                             level 11 row 2,                     advanced one pass,                  135119
+github,                             level 11 row 2,                     advanced one pass,                  135367
 github,                             level 11 row 2 with dict dms,       advanced one pass,                  39671
 github,                             level 11 row 2 with dict dds,       advanced one pass,                  39671
 github,                             level 11 row 2 with dict copy,      advanced one pass,                  39651
 github,                             level 11 row 2 with dict load,      advanced one pass,                  41360
-github,                             level 12 row 1,                     advanced one pass,                  134180
+github,                             level 12 row 1,                     advanced one pass,                  134402
 github,                             level 12 row 1 with dict dms,       advanced one pass,                  39677
 github,                             level 12 row 1 with dict dds,       advanced one pass,                  39677
 github,                             level 12 row 1 with dict copy,      advanced one pass,                  39677
 github,                             level 12 row 1 with dict load,      advanced one pass,                  41166
-github,                             level 12 row 2,                     advanced one pass,                  134180
+github,                             level 12 row 2,                     advanced one pass,                  134402
 github,                             level 12 row 2 with dict dms,       advanced one pass,                  39677
 github,                             level 12 row 2 with dict dds,       advanced one pass,                  39677
 github,                             level 12 row 2 with dict copy,      advanced one pass,                  39677
 github,                             level 12 row 2 with dict load,      advanced one pass,                  41166
-github,                             level 13,                           advanced one pass,                  132729
+github,                             level 13,                           advanced one pass,                  132878
 github,                             level 13 with dict,                 advanced one pass,                  39900
 github,                             level 13 with dict dms,             advanced one pass,                  39900
 github,                             level 13 with dict dds,             advanced one pass,                  39900
 github,                             level 13 with dict copy,            advanced one pass,                  39948
 github,                             level 13 with dict load,            advanced one pass,                  42624
-github,                             level 16,                           advanced one pass,                  132729
+github,                             level 16,                           advanced one pass,                  133209
 github,                             level 16 with dict,                 advanced one pass,                  37577
 github,                             level 16 with dict dms,             advanced one pass,                  37577
 github,                             level 16 with dict dds,             advanced one pass,                  37577
 github,                             level 16 with dict copy,            advanced one pass,                  37568
 github,                             level 16 with dict load,            advanced one pass,                  42338
-github,                             level 19,                           advanced one pass,                  132729
+github,                             level 19,                           advanced one pass,                  132879
 github,                             level 19 with dict,                 advanced one pass,                  37576
 github,                             level 19 with dict dms,             advanced one pass,                  37576
 github,                             level 19 with dict dds,             advanced one pass,                  37576
@@ -418,7 +418,7 @@ github,                             small hash log,                     advanced
 github,                             small chain log,                    advanced one pass,                  136341
 github,                             explicit params,                    advanced one pass,                  137727
 github,                             uncompressed literals,              advanced one pass,                  165911
-github,                             uncompressed literals optimal,      advanced one pass,                  157227
+github,                             uncompressed literals optimal,      advanced one pass,                  152667
 github,                             huffman literals,                   advanced one pass,                  142365
 github,                             multithreaded with advanced params, advanced one pass,                  165911
 github.tar,                         level -5,                           advanced one pass,                  52115
@@ -521,13 +521,13 @@ github.tar,                         level 13 with dict dms,             advanced
 github.tar,                         level 13 with dict dds,             advanced one pass,                  37220
 github.tar,                         level 13 with dict copy,            advanced one pass,                  37130
 github.tar,                         level 13 with dict load,            advanced one pass,                  36010
-github.tar,                         level 16,                           advanced one pass,                  40471
+github.tar,                         level 16,                           advanced one pass,                  40466
 github.tar,                         level 16 with dict,                 advanced one pass,                  33374
 github.tar,                         level 16 with dict dms,             advanced one pass,                  33206
 github.tar,                         level 16 with dict dds,             advanced one pass,                  33206
 github.tar,                         level 16 with dict copy,            advanced one pass,                  33374
 github.tar,                         level 16 with dict load,            advanced one pass,                  39081
-github.tar,                         level 19,                           advanced one pass,                  32149
+github.tar,                         level 19,                           advanced one pass,                  32276
 github.tar,                         level 19 with dict,                 advanced one pass,                  32712
 github.tar,                         level 19 with dict dms,             advanced one pass,                  32555
 github.tar,                         level 19 with dict dds,             advanced one pass,                  32555
@@ -566,8 +566,8 @@ silesia,                            level 11 row 2,                     advanced
 silesia,                            level 12 row 1,                     advanced one pass small out,        4505046
 silesia,                            level 12 row 2,                     advanced one pass small out,        4503116
 silesia,                            level 13,                           advanced one pass small out,        4493990
-silesia,                            level 16,                           advanced one pass small out,        4359864
-silesia,                            level 19,                           advanced one pass small out,        4296438
+silesia,                            level 16,                           advanced one pass small out,        4360041
+silesia,                            level 19,                           advanced one pass small out,        4296055
 silesia,                            no source size,                     advanced one pass small out,        4842075
 silesia,                            long distance mode,                 advanced one pass small out,        4833710
 silesia,                            multithreaded,                      advanced one pass small out,        4842075
@@ -600,14 +600,14 @@ silesia.tar,                        level 11 row 2,                     advanced
 silesia.tar,                        level 12 row 1,                     advanced one pass small out,        4514049
 silesia.tar,                        level 12 row 2,                     advanced one pass small out,        4513797
 silesia.tar,                        level 13,                           advanced one pass small out,        4502956
-silesia.tar,                        level 16,                           advanced one pass small out,        4360527
-silesia.tar,                        level 19,                           advanced one pass small out,        4266970
+silesia.tar,                        level 16,                           advanced one pass small out,        4360546
+silesia.tar,                        level 19,                           advanced one pass small out,        4265911
 silesia.tar,                        no source size,                     advanced one pass small out,        4854086
 silesia.tar,                        long distance mode,                 advanced one pass small out,        4840452
 silesia.tar,                        multithreaded,                      advanced one pass small out,        4854160
 silesia.tar,                        multithreaded long distance mode,   advanced one pass small out,        4845741
 silesia.tar,                        small window log,                   advanced one pass small out,        7100655
-silesia.tar,                        small hash log,                     advanced one pass small out,        6529231
+silesia.tar,                        small hash log,                     advanced one pass small out,        6529206
 silesia.tar,                        small chain log,                    advanced one pass small out,        4917041
 silesia.tar,                        explicit params,                    advanced one pass small out,        4806855
 silesia.tar,                        uncompressed literals,              advanced one pass small out,        5122473
@@ -688,39 +688,39 @@ github,                             level 9 with dict dms,              advanced
 github,                             level 9 with dict dds,              advanced one pass small out,        39393
 github,                             level 9 with dict copy,             advanced one pass small out,        39398
 github,                             level 9 with dict load,             advanced one pass small out,        41710
-github,                             level 11 row 1,                     advanced one pass small out,        135119
+github,                             level 11 row 1,                     advanced one pass small out,        135367
 github,                             level 11 row 1 with dict dms,       advanced one pass small out,        39671
 github,                             level 11 row 1 with dict dds,       advanced one pass small out,        39671
 github,                             level 11 row 1 with dict copy,      advanced one pass small out,        39651
 github,                             level 11 row 1 with dict load,      advanced one pass small out,        41360
-github,                             level 11 row 2,                     advanced one pass small out,        135119
+github,                             level 11 row 2,                     advanced one pass small out,        135367
 github,                             level 11 row 2 with dict dms,       advanced one pass small out,        39671
 github,                             level 11 row 2 with dict dds,       advanced one pass small out,        39671
 github,                             level 11 row 2 with dict copy,      advanced one pass small out,        39651
 github,                             level 11 row 2 with dict load,      advanced one pass small out,        41360
-github,                             level 12 row 1,                     advanced one pass small out,        134180
+github,                             level 12 row 1,                     advanced one pass small out,        134402
 github,                             level 12 row 1 with dict dms,       advanced one pass small out,        39677
 github,                             level 12 row 1 with dict dds,       advanced one pass small out,        39677
 github,                             level 12 row 1 with dict copy,      advanced one pass small out,        39677
 github,                             level 12 row 1 with dict load,      advanced one pass small out,        41166
-github,                             level 12 row 2,                     advanced one pass small out,        134180
+github,                             level 12 row 2,                     advanced one pass small out,        134402
 github,                             level 12 row 2 with dict dms,       advanced one pass small out,        39677
 github,                             level 12 row 2 with dict dds,       advanced one pass small out,        39677
 github,                             level 12 row 2 with dict copy,      advanced one pass small out,        39677
 github,                             level 12 row 2 with dict load,      advanced one pass small out,        41166
-github,                             level 13,                           advanced one pass small out,        132729
+github,                             level 13,                           advanced one pass small out,        132878
 github,                             level 13 with dict,                 advanced one pass small out,        39900
 github,                             level 13 with dict dms,             advanced one pass small out,        39900
 github,                             level 13 with dict dds,             advanced one pass small out,        39900
 github,                             level 13 with dict copy,            advanced one pass small out,        39948
 github,                             level 13 with dict load,            advanced one pass small out,        42624
-github,                             level 16,                           advanced one pass small out,        132729
+github,                             level 16,                           advanced one pass small out,        133209
 github,                             level 16 with dict,                 advanced one pass small out,        37577
 github,                             level 16 with dict dms,             advanced one pass small out,        37577
 github,                             level 16 with dict dds,             advanced one pass small out,        37577
 github,                             level 16 with dict copy,            advanced one pass small out,        37568
 github,                             level 16 with dict load,            advanced one pass small out,        42338
-github,                             level 19,                           advanced one pass small out,        132729
+github,                             level 19,                           advanced one pass small out,        132879
 github,                             level 19 with dict,                 advanced one pass small out,        37576
 github,                             level 19 with dict dms,             advanced one pass small out,        37576
 github,                             level 19 with dict dds,             advanced one pass small out,        37576
@@ -736,7 +736,7 @@ github,                             small hash log,                     advanced
 github,                             small chain log,                    advanced one pass small out,        136341
 github,                             explicit params,                    advanced one pass small out,        137727
 github,                             uncompressed literals,              advanced one pass small out,        165911
-github,                             uncompressed literals optimal,      advanced one pass small out,        157227
+github,                             uncompressed literals optimal,      advanced one pass small out,        152667
 github,                             huffman literals,                   advanced one pass small out,        142365
 github,                             multithreaded with advanced params, advanced one pass small out,        165911
 github.tar,                         level -5,                           advanced one pass small out,        52115
@@ -839,13 +839,13 @@ github.tar,                         level 13 with dict dms,             advanced
 github.tar,                         level 13 with dict dds,             advanced one pass small out,        37220
 github.tar,                         level 13 with dict copy,            advanced one pass small out,        37130
 github.tar,                         level 13 with dict load,            advanced one pass small out,        36010
-github.tar,                         level 16,                           advanced one pass small out,        40471
+github.tar,                         level 16,                           advanced one pass small out,        40466
 github.tar,                         level 16 with dict,                 advanced one pass small out,        33374
 github.tar,                         level 16 with dict dms,             advanced one pass small out,        33206
 github.tar,                         level 16 with dict dds,             advanced one pass small out,        33206
 github.tar,                         level 16 with dict copy,            advanced one pass small out,        33374
 github.tar,                         level 16 with dict load,            advanced one pass small out,        39081
-github.tar,                         level 19,                           advanced one pass small out,        32149
+github.tar,                         level 19,                           advanced one pass small out,        32276
 github.tar,                         level 19 with dict,                 advanced one pass small out,        32712
 github.tar,                         level 19 with dict dms,             advanced one pass small out,        32555
 github.tar,                         level 19 with dict dds,             advanced one pass small out,        32555
@@ -884,8 +884,8 @@ silesia,                            level 11 row 2,                     advanced
 silesia,                            level 12 row 1,                     advanced streaming,                 4505046
 silesia,                            level 12 row 2,                     advanced streaming,                 4503116
 silesia,                            level 13,                           advanced streaming,                 4493990
-silesia,                            level 16,                           advanced streaming,                 4359864
-silesia,                            level 19,                           advanced streaming,                 4296438
+silesia,                            level 16,                           advanced streaming,                 4360041
+silesia,                            level 19,                           advanced streaming,                 4296055
 silesia,                            no source size,                     advanced streaming,                 4842039
 silesia,                            long distance mode,                 advanced streaming,                 4833710
 silesia,                            multithreaded,                      advanced streaming,                 4842075
@@ -918,14 +918,14 @@ silesia.tar,                        level 11 row 2,                     advanced
 silesia.tar,                        level 12 row 1,                     advanced streaming,                 4514049
 silesia.tar,                        level 12 row 2,                     advanced streaming,                 4513797
 silesia.tar,                        level 13,                           advanced streaming,                 4502956
-silesia.tar,                        level 16,                           advanced streaming,                 4360527
-silesia.tar,                        level 19,                           advanced streaming,                 4266970
+silesia.tar,                        level 16,                           advanced streaming,                 4360546
+silesia.tar,                        level 19,                           advanced streaming,                 4265911
 silesia.tar,                        no source size,                     advanced streaming,                 4859267
 silesia.tar,                        long distance mode,                 advanced streaming,                 4840452
 silesia.tar,                        multithreaded,                      advanced streaming,                 4854160
 silesia.tar,                        multithreaded long distance mode,   advanced streaming,                 4845741
 silesia.tar,                        small window log,                   advanced streaming,                 7117559
-silesia.tar,                        small hash log,                     advanced streaming,                 6529234
+silesia.tar,                        small hash log,                     advanced streaming,                 6529209
 silesia.tar,                        small chain log,                    advanced streaming,                 4917021
 silesia.tar,                        explicit params,                    advanced streaming,                 4806873
 silesia.tar,                        uncompressed literals,              advanced streaming,                 5127423
@@ -1006,39 +1006,39 @@ github,                             level 9 with dict dms,              advanced
 github,                             level 9 with dict dds,              advanced streaming,                 39393
 github,                             level 9 with dict copy,             advanced streaming,                 39398
 github,                             level 9 with dict load,             advanced streaming,                 41710
-github,                             level 11 row 1,                     advanced streaming,                 135119
+github,                             level 11 row 1,                     advanced streaming,                 135367
 github,                             level 11 row 1 with dict dms,       advanced streaming,                 39671
 github,                             level 11 row 1 with dict dds,       advanced streaming,                 39671
 github,                             level 11 row 1 with dict copy,      advanced streaming,                 39651
 github,                             level 11 row 1 with dict load,      advanced streaming,                 41360
-github,                             level 11 row 2,                     advanced streaming,                 135119
+github,                             level 11 row 2,                     advanced streaming,                 135367
 github,                             level 11 row 2 with dict dms,       advanced streaming,                 39671
 github,                             level 11 row 2 with dict dds,       advanced streaming,                 39671
 github,                             level 11 row 2 with dict copy,      advanced streaming,                 39651
 github,                             level 11 row 2 with dict load,      advanced streaming,                 41360
-github,                             level 12 row 1,                     advanced streaming,                 134180
+github,                             level 12 row 1,                     advanced streaming,                 134402
 github,                             level 12 row 1 with dict dms,       advanced streaming,                 39677
 github,                             level 12 row 1 with dict dds,       advanced streaming,                 39677
 github,                             level 12 row 1 with dict copy,      advanced streaming,                 39677
 github,                             level 12 row 1 with dict load,      advanced streaming,                 41166
-github,                             level 12 row 2,                     advanced streaming,                 134180
+github,                             level 12 row 2,                     advanced streaming,                 134402
 github,                             level 12 row 2 with dict dms,       advanced streaming,                 39677
 github,                             level 12 row 2 with dict dds,       advanced streaming,                 39677
 github,                             level 12 row 2 with dict copy,      advanced streaming,                 39677
 github,                             level 12 row 2 with dict load,      advanced streaming,                 41166
-github,                             level 13,                           advanced streaming,                 132729
+github,                             level 13,                           advanced streaming,                 132878
 github,                             level 13 with dict,                 advanced streaming,                 39900
 github,                             level 13 with dict dms,             advanced streaming,                 39900
 github,                             level 13 with dict dds,             advanced streaming,                 39900
 github,                             level 13 with dict copy,            advanced streaming,                 39948
 github,                             level 13 with dict load,            advanced streaming,                 42624
-github,                             level 16,                           advanced streaming,                 132729
+github,                             level 16,                           advanced streaming,                 133209
 github,                             level 16 with dict,                 advanced streaming,                 37577
 github,                             level 16 with dict dms,             advanced streaming,                 37577
 github,                             level 16 with dict dds,             advanced streaming,                 37577
 github,                             level 16 with dict copy,            advanced streaming,                 37568
 github,                             level 16 with dict load,            advanced streaming,                 42338
-github,                             level 19,                           advanced streaming,                 132729
+github,                             level 19,                           advanced streaming,                 132879
 github,                             level 19 with dict,                 advanced streaming,                 37576
 github,                             level 19 with dict dms,             advanced streaming,                 37576
 github,                             level 19 with dict dds,             advanced streaming,                 37576
@@ -1054,7 +1054,7 @@ github,                             small hash log,                     advanced
 github,                             small chain log,                    advanced streaming,                 136341
 github,                             explicit params,                    advanced streaming,                 137727
 github,                             uncompressed literals,              advanced streaming,                 165911
-github,                             uncompressed literals optimal,      advanced streaming,                 157227
+github,                             uncompressed literals optimal,      advanced streaming,                 152667
 github,                             huffman literals,                   advanced streaming,                 142365
 github,                             multithreaded with advanced params, advanced streaming,                 165911
 github.tar,                         level -5,                           advanced streaming,                 52152
@@ -1157,13 +1157,13 @@ github.tar,                         level 13 with dict dms,             advanced
 github.tar,                         level 13 with dict dds,             advanced streaming,                 37220
 github.tar,                         level 13 with dict copy,            advanced streaming,                 37130
 github.tar,                         level 13 with dict load,            advanced streaming,                 36010
-github.tar,                         level 16,                           advanced streaming,                 40471
+github.tar,                         level 16,                           advanced streaming,                 40466
 github.tar,                         level 16 with dict,                 advanced streaming,                 33374
 github.tar,                         level 16 with dict dms,             advanced streaming,                 33206
 github.tar,                         level 16 with dict dds,             advanced streaming,                 33206
 github.tar,                         level 16 with dict copy,            advanced streaming,                 33374
 github.tar,                         level 16 with dict load,            advanced streaming,                 39081
-github.tar,                         level 19,                           advanced streaming,                 32149
+github.tar,                         level 19,                           advanced streaming,                 32276
 github.tar,                         level 19 with dict,                 advanced streaming,                 32712
 github.tar,                         level 19 with dict dms,             advanced streaming,                 32555
 github.tar,                         level 19 with dict dds,             advanced streaming,                 32555
@@ -1194,11 +1194,11 @@ silesia,                            level 6,                            old stre
 silesia,                            level 7,                            old streaming,                      4566984
 silesia,                            level 9,                            old streaming,                      4543018
 silesia,                            level 13,                           old streaming,                      4493990
-silesia,                            level 16,                           old streaming,                      4359864
-silesia,                            level 19,                           old streaming,                      4296438
+silesia,                            level 16,                           old streaming,                      4360041
+silesia,                            level 19,                           old streaming,                      4296055
 silesia,                            no source size,                     old streaming,                      4842039
 silesia,                            uncompressed literals,              old streaming,                      4842075
-silesia,                            uncompressed literals optimal,      old streaming,                      4296438
+silesia,                            uncompressed literals optimal,      old streaming,                      4296055
 silesia,                            huffman literals,                   old streaming,                      6172207
 silesia.tar,                        level -5,                           old streaming,                      6856523
 silesia.tar,                        level -3,                           old streaming,                      6505954
@@ -1212,11 +1212,11 @@ silesia.tar,                        level 6,                            old stre
 silesia.tar,                        level 7,                            old streaming,                      4576664
 silesia.tar,                        level 9,                            old streaming,                      4552900
 silesia.tar,                        level 13,                           old streaming,                      4502956
-silesia.tar,                        level 16,                           old streaming,                      4360527
-silesia.tar,                        level 19,                           old streaming,                      4266970
+silesia.tar,                        level 16,                           old streaming,                      4360546
+silesia.tar,                        level 19,                           old streaming,                      4265911
 silesia.tar,                        no source size,                     old streaming,                      4859267
 silesia.tar,                        uncompressed literals,              old streaming,                      4859271
-silesia.tar,                        uncompressed literals optimal,      old streaming,                      4266970
+silesia.tar,                        uncompressed literals optimal,      old streaming,                      4265911
 silesia.tar,                        huffman literals,                   old streaming,                      6179056
 github,                             level -5,                           old streaming,                      204407
 github,                             level -5 with dict,                 old streaming,                      46718
@@ -1240,16 +1240,16 @@ github,                             level 7,                            old stre
 github,                             level 7 with dict,                  old streaming,                      38758
 github,                             level 9,                            old streaming,                      135122
 github,                             level 9 with dict,                  old streaming,                      39437
-github,                             level 13,                           old streaming,                      132729
+github,                             level 13,                           old streaming,                      132878
 github,                             level 13 with dict,                 old streaming,                      39900
-github,                             level 16,                           old streaming,                      132729
+github,                             level 16,                           old streaming,                      133209
 github,                             level 16 with dict,                 old streaming,                      37577
-github,                             level 19,                           old streaming,                      132729
+github,                             level 19,                           old streaming,                      132879
 github,                             level 19 with dict,                 old streaming,                      37576
 github,                             no source size,                     old streaming,                      140599
 github,                             no source size with dict,           old streaming,                      40654
 github,                             uncompressed literals,              old streaming,                      136332
-github,                             uncompressed literals optimal,      old streaming,                      132729
+github,                             uncompressed literals optimal,      old streaming,                      132879
 github,                             huffman literals,                   old streaming,                      175468
 github.tar,                         level -5,                           old streaming,                      52152
 github.tar,                         level -5 with dict,                 old streaming,                      51045
@@ -1275,14 +1275,14 @@ github.tar,                         level 9,                            old stre
 github.tar,                         level 9 with dict,                  old streaming,                      36484
 github.tar,                         level 13,                           old streaming,                      35501
 github.tar,                         level 13 with dict,                 old streaming,                      37130
-github.tar,                         level 16,                           old streaming,                      40471
+github.tar,                         level 16,                           old streaming,                      40466
 github.tar,                         level 16 with dict,                 old streaming,                      33374
-github.tar,                         level 19,                           old streaming,                      32149
+github.tar,                         level 19,                           old streaming,                      32276
 github.tar,                         level 19 with dict,                 old streaming,                      32712
 github.tar,                         no source size,                     old streaming,                      38828
 github.tar,                         no source size with dict,           old streaming,                      38000
 github.tar,                         uncompressed literals,              old streaming,                      38831
-github.tar,                         uncompressed literals optimal,      old streaming,                      32149
+github.tar,                         uncompressed literals optimal,      old streaming,                      32276
 github.tar,                         huffman literals,                   old streaming,                      42560
 silesia,                            level -5,                           old streaming advanced,             6854744
 silesia,                            level -3,                           old streaming advanced,             6503319
@@ -1296,8 +1296,8 @@ silesia,                            level 6,                            old stre
 silesia,                            level 7,                            old streaming advanced,             4566984
 silesia,                            level 9,                            old streaming advanced,             4543018
 silesia,                            level 13,                           old streaming advanced,             4493990
-silesia,                            level 16,                           old streaming advanced,             4359864
-silesia,                            level 19,                           old streaming advanced,             4296438
+silesia,                            level 16,                           old streaming advanced,             4360041
+silesia,                            level 19,                           old streaming advanced,             4296055
 silesia,                            no source size,                     old streaming advanced,             4842039
 silesia,                            long distance mode,                 old streaming advanced,             4842075
 silesia,                            multithreaded,                      old streaming advanced,             4842075
@@ -1307,7 +1307,7 @@ silesia,                            small hash log,                     old stre
 silesia,                            small chain log,                    old streaming advanced,             4912197
 silesia,                            explicit params,                    old streaming advanced,             4795452
 silesia,                            uncompressed literals,              old streaming advanced,             4842075
-silesia,                            uncompressed literals optimal,      old streaming advanced,             4296438
+silesia,                            uncompressed literals optimal,      old streaming advanced,             4296055
 silesia,                            huffman literals,                   old streaming advanced,             6172207
 silesia,                            multithreaded with advanced params, old streaming advanced,             4842075
 silesia.tar,                        level -5,                           old streaming advanced,             6856523
@@ -1322,18 +1322,18 @@ silesia.tar,                        level 6,                            old stre
 silesia.tar,                        level 7,                            old streaming advanced,             4576664
 silesia.tar,                        level 9,                            old streaming advanced,             4552900
 silesia.tar,                        level 13,                           old streaming advanced,             4502956
-silesia.tar,                        level 16,                           old streaming advanced,             4360527
-silesia.tar,                        level 19,                           old streaming advanced,             4266970
+silesia.tar,                        level 16,                           old streaming advanced,             4360546
+silesia.tar,                        level 19,                           old streaming advanced,             4265911
 silesia.tar,                        no source size,                     old streaming advanced,             4859267
 silesia.tar,                        long distance mode,                 old streaming advanced,             4859271
 silesia.tar,                        multithreaded,                      old streaming advanced,             4859271
 silesia.tar,                        multithreaded long distance mode,   old streaming advanced,             4859271
 silesia.tar,                        small window log,                   old streaming advanced,             7117562
-silesia.tar,                        small hash log,                     old streaming advanced,             6529234
+silesia.tar,                        small hash log,                     old streaming advanced,             6529209
 silesia.tar,                        small chain log,                    old streaming advanced,             4917021
 silesia.tar,                        explicit params,                    old streaming advanced,             4806873
 silesia.tar,                        uncompressed literals,              old streaming advanced,             4859271
-silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4266970
+silesia.tar,                        uncompressed literals optimal,      old streaming advanced,             4265911
 silesia.tar,                        huffman literals,                   old streaming advanced,             6179056
 silesia.tar,                        multithreaded with advanced params, old streaming advanced,             4859271
 github,                             level -5,                           old streaming advanced,             213265
@@ -1360,9 +1360,9 @@ github,                             level 9,                            old stre
 github,                             level 9 with dict,                  old streaming advanced,             38981
 github,                             level 13,                           old streaming advanced,             138676
 github,                             level 13 with dict,                 old streaming advanced,             39725
-github,                             level 16,                           old streaming advanced,             138676
+github,                             level 16,                           old streaming advanced,             138575
 github,                             level 16 with dict,                 old streaming advanced,             40789
-github,                             level 19,                           old streaming advanced,             132729
+github,                             level 19,                           old streaming advanced,             132879
 github,                             level 19 with dict,                 old streaming advanced,             37576
 github,                             no source size,                     old streaming advanced,             140599
 github,                             no source size with dict,           old streaming advanced,             40608
@@ -1374,7 +1374,7 @@ github,                             small hash log,                     old stre
 github,                             small chain log,                    old streaming advanced,             139275
 github,                             explicit params,                    old streaming advanced,             140937
 github,                             uncompressed literals,              old streaming advanced,             141104
-github,                             uncompressed literals optimal,      old streaming advanced,             132729
+github,                             uncompressed literals optimal,      old streaming advanced,             132879
 github,                             huffman literals,                   old streaming advanced,             181107
 github,                             multithreaded with advanced params, old streaming advanced,             141104
 github.tar,                         level -5,                           old streaming advanced,             52152
@@ -1401,9 +1401,9 @@ github.tar,                         level 9,                            old stre
 github.tar,                         level 9 with dict,                  old streaming advanced,             36312
 github.tar,                         level 13,                           old streaming advanced,             35501
 github.tar,                         level 13 with dict,                 old streaming advanced,             35807
-github.tar,                         level 16,                           old streaming advanced,             40471
+github.tar,                         level 16,                           old streaming advanced,             40466
 github.tar,                         level 16 with dict,                 old streaming advanced,             38578
-github.tar,                         level 19,                           old streaming advanced,             32149
+github.tar,                         level 19,                           old streaming advanced,             32276
 github.tar,                         level 19 with dict,                 old streaming advanced,             32704
 github.tar,                         no source size,                     old streaming advanced,             38828
 github.tar,                         no source size with dict,           old streaming advanced,             38015
@@ -1415,7 +1415,7 @@ github.tar,                         small hash log,                     old stre
 github.tar,                         small chain log,                    old streaming advanced,             41669
 github.tar,                         explicit params,                    old streaming advanced,             41385
 github.tar,                         uncompressed literals,              old streaming advanced,             38831
-github.tar,                         uncompressed literals optimal,      old streaming advanced,             32149
+github.tar,                         uncompressed literals optimal,      old streaming advanced,             32276
 github.tar,                         huffman literals,                   old streaming advanced,             42560
 github.tar,                         multithreaded with advanced params, old streaming advanced,             38831
 github,                             level -5 with dict,                 old streaming cdict,                46718
diff --git a/tests/regression/test.c b/tests/regression/test.c
index 1de6be86c..07600be57 100644
--- a/tests/regression/test.c
+++ b/tests/regression/test.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c
index 9aa208cff..77411cd49 100644
--- a/tests/roundTripCrash.c
+++ b/tests/roundTripCrash.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/seqgen.c b/tests/seqgen.c
index 1e340c851..0d8a766c8 100644
--- a/tests/seqgen.c
+++ b/tests/seqgen.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/seqgen.h b/tests/seqgen.h
index cea3f552b..df17398ef 100644
--- a/tests/seqgen.h
+++ b/tests/seqgen.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/tests/test-license.py b/tests/test-license.py
index f27cb6774..d54c16419 100755
--- a/tests/test-license.py
+++ b/tests/test-license.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -83,8 +83,8 @@ def valid_copyright(lines):
             continue
         if "present" in line:
             return (False, f"Copyright line '{line}' contains 'present'!")
-        if "Facebook, Inc" not in line:
-            return (False, f"Copyright line '{line}' does not contain 'Facebook, Inc'")
+        if "Meta Platforms, Inc" not in line:
+            return (False, f"Copyright line '{line}' does not contain 'Meta Platforms, Inc'")
         year = YEAR_REGEX.search(line)
         if year is not None:
             return (False, f"Copyright line '{line}' contains {year.group(0)}; it should be yearless")
diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py
index baca251f5..1bcf39e2b 100755
--- a/tests/test-zstd-versions.py
+++ b/tests/test-zstd-versions.py
@@ -2,7 +2,7 @@
 """Test zstd interoperability between versions"""
 
 # ################################################################
-# Copyright (c) Yann Collet, Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -23,14 +23,25 @@ from subprocess import Popen, PIPE
 repo_url = 'https://github.com/facebook/zstd.git'
 tmp_dir_name = 'tests/versionsTest'
 make_cmd = 'make'
-make_args = ['-j','CFLAGS=-O1']
+make_args = ['-j','CFLAGS=-O0']
 git_cmd = 'git'
 test_dat_src = 'README.md'
 test_dat = 'test_dat'
 head = 'vdevel'
 dict_source = 'dict_source'
-dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
-dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
+dict_globs = [
+    'programs/*.c',
+    'lib/common/*.c',
+    'lib/compress/*.c',
+    'lib/decompress/*.c',
+    'lib/dictBuilder/*.c',
+    'lib/legacy/*.c',
+    'programs/*.h',
+    'lib/common/*.h',
+    'lib/compress/*.h',
+    'lib/dictBuilder/*.h',
+    'lib/legacy/*.h'
+]
 
 
 def execute(command, print_output=False, print_error=True, param_shell=False):
@@ -74,59 +85,85 @@ def get_git_tags():
     return tags
 
 
-def create_dict(tag, dict_source_path):
+def dict_ok(tag, dict_name, sample):
+    if not os.path.isfile(dict_name):
+        return False
+    try:
+        cmd = ['./zstd.' + tag, '-D', dict_name]
+        with open(sample, "rb") as i:
+            subprocess.check_call(cmd, stdin=i, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        return True
+    except:
+        return False
+
+
+def create_dict(tag, dict_source_path, fallback_tag=None):
     dict_name = 'dict.' + tag
     if not os.path.isfile(dict_name):
         cFiles = glob.glob(dict_source_path + "/*.c")
         hFiles = glob.glob(dict_source_path + "/*.h")
+        # Ensure the dictionary builder is deterministic
+        files = sorted(cFiles + hFiles)
         if tag == 'v0.5.0':
-            result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
+            result = execute('./dictBuilder.' + tag + ' ' + ' '.join(files) + ' -o ' + dict_name, print_output=False, param_shell=True)
         else:
-            result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
-        if result == 0:
+            result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(files) + ' -o ' + dict_name, print_output=False, param_shell=True)
+        if result == 0 and dict_ok(tag, dict_name, files[0]):
             print(dict_name + ' created')
+        elif fallback_tag is not None:
+            fallback_dict_name = 'dict.' + fallback_tag
+            print('creating dictionary ' + dict_name + ' failed, falling back to ' + fallback_dict_name)
+            shutil.copy(fallback_dict_name, dict_name)
         else:
-            print('ERROR: creating of ' + dict_name + ' failed')
+            raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
     else:
         print(dict_name + ' already exists')
 
 
+def zstd(tag, args, input_file, output_file):
+    """
+    Zstd compress input_file to output_file.
+    Need this helper because 0.5.0 is broken when stdout is not a TTY.
+    Throws an exception if the command returns non-zero.
+    """
+    with open(input_file, "rb") as i:
+        with open(output_file, "wb") as o:
+            cmd = ['./zstd.' + tag] + args
+            print("Running: '{}', input={}, output={}" .format(
+                ' '.join(cmd), input_file, output_file
+            ))
+            result = subprocess.run(cmd, stdin=i, stdout=o, stderr=subprocess.PIPE)
+            print("Stderr: {}".format(result.stderr.decode("ascii")))
+            result.check_returncode()
+
+
 def dict_compress_sample(tag, sample):
     dict_name = 'dict.' + tag
-    DEVNULL = open(os.devnull, 'wb')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f',   sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
-    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
+    verbose = ['-v', '-v', '-v']
+    zstd(tag, ['-D', dict_name, '-1'] + verbose, sample, sample + '_01_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-3'], sample, sample + '_03_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-5'], sample, sample + '_05_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-9'], sample, sample + '_09_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-15'], sample, sample + '_15_64_' + tag + '_dictio.zst')
+    zstd(tag, ['-D', dict_name, '-18'], sample, sample + '_18_64_' + tag + '_dictio.zst')
     # zstdFiles = glob.glob("*.zst*")
     # print(zstdFiles)
     print(tag + " : dict compression completed")
 
 
 def compress_sample(tag, sample):
-    DEVNULL = open(os.devnull, 'wb')
-    if subprocess.call(['./zstd.' + tag, '-f',   sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-5f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-9f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst')
-    if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-1'], sample, sample + '_01_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-3'], sample, sample + '_03_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-5'], sample, sample + '_05_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-9'], sample, sample + '_09_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-15'], sample, sample + '_15_64_' + tag + '_nodict.zst')
+    zstd(tag, ['-18'], sample, sample + '_18_64_' + tag + '_nodict.zst')
     # zstdFiles = glob.glob("*.zst*")
     # print(zstdFiles)
     print(tag + " : compression completed")
 
 
-# http://stackoverflow.com/a/19711609/2132223
+# https://stackoverflow.com/a/19711609/2132223
 def sha1_of_file(filepath):
     with open(filepath, 'rb') as f:
         return hashlib.sha1(f.read()).hexdigest()
@@ -150,23 +187,13 @@ def decompress_zst(tag):
     dec_error = 0
     list_zst = sorted(glob.glob('*_nodict.zst'))
     for file_zst in list_zst:
-        print(file_zst, end=' ')
-        print(tag, end=' ')
+        print(file_zst + ' ' + tag)
         file_dec = file_zst + '_d64_' + tag + '.dec'
-        if tag <= 'v0.5.0':
-            params = ['./zstd.' + tag, '-df', file_zst, file_dec]
+        zstd(tag, ['-d'], file_zst, file_dec)
+        if not filecmp.cmp(file_dec, test_dat):
+            raise RuntimeError('Decompression failed: tag={} file={}'.format(tag, file_zst))
         else:
-            params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
-        if execute(params) == 0:
-            if not filecmp.cmp(file_dec, test_dat):
-                print('ERR !! ')
-                dec_error = 1
-            else:
-                print('OK     ')
-        else:
-            print('command does not work')
-            dec_error = 1
-    return dec_error
+            print('OK     ')
 
 
 def decompress_dict(tag):
@@ -181,22 +208,13 @@ def decompress_dict(tag):
         if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
             continue
         dict_name = 'dict.' + dict_tag
-        print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ')
+        print(file_zst + ' ' + tag + ' dict=' + dict_tag)
         file_dec = file_zst + '_d64_' + tag + '.dec'
-        if tag <= 'v0.5.0':
-            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec]
+        zstd(tag, ['-D', dict_name, '-d'], file_zst, file_dec)
+        if not filecmp.cmp(file_dec, test_dat):
+            raise RuntimeError('Decompression failed: tag={} file={}'.format(tag, file_zst))
         else:
-            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec]
-        if execute(params) == 0:
-            if not filecmp.cmp(file_dec, test_dat):
-                print('ERR !! ')
-                dec_error = 1
-            else:
-                print('OK     ')
-        else:
-            print('command does not work')
-            dec_error = 1
-    return dec_error
+            print('OK     ')
 
 
 if __name__ == '__main__':
@@ -260,32 +278,31 @@ if __name__ == '__main__':
     # copy *.c and *.h to a temporary directory ("dict_source")
     if not os.path.isdir(dict_source_path):
         os.mkdir(dict_source_path)
-        print('cp ' + dict_files + ' ' + dict_source_path)
-        execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
+        for dict_glob in dict_globs:
+            files = glob.glob(dict_glob, root_dir=base_dir)
+            for file in files:
+                file = os.path.join(base_dir, file)
+                print("copying " + file + " to " + dict_source_path)
+                shutil.copy(file, dict_source_path)
 
     print('-----------------------------------------------')
     print('Compress test.dat by all released zstd')
     print('-----------------------------------------------')
 
-    error_code = 0
+    create_dict(head, dict_source_path)
     for tag in tags:
         print(tag)
         if tag >= 'v0.5.0':
-            create_dict(tag, dict_source_path)
+            create_dict(tag, dict_source_path, head)
             dict_compress_sample(tag, test_dat)
             remove_duplicates()
-            error_code += decompress_dict(tag)
+            decompress_dict(tag)
         compress_sample(tag, test_dat)
         remove_duplicates()
-        error_code += decompress_zst(tag)
+        decompress_zst(tag)
 
     print('')
     print('Enumerate different compressed files')
     zstds = sorted(glob.glob('*.zst'))
     for zstd in zstds:
         print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd))
-
-    if error_code != 0:
-        print('======  ERROR !!!  =======')
-
-    sys.exit(error_code)
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 348f72ed4..aff847b4d 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -25,6 +25,7 @@
 #include        /* free */
 #include         /* fgets, sscanf */
 #include        /* strcmp */
+#include          /* time_t, time(), to randomize seed */
 #include        /* assert */
 #include "timefn.h"       /* UTIL_time_t, UTIL_getTime */
 #include "mem.h"
@@ -39,7 +40,7 @@
 #include "seqgen.h"
 #include "util.h"
 #include "timefn.h"       /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
-
+#include "external_matchfinder.h"   /* zstreamExternalMatchFinder, EMF_testCase */
 
 /*-************************************
  *  Constants
@@ -259,7 +260,7 @@ static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
     return 0;
 }
 
-static int basicUnitTests(U32 seed, double compressibility)
+static int basicUnitTests(U32 seed, double compressibility, int bigTests)
 {
     size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
     void* CNBuffer = malloc(CNBufferSize);
@@ -1565,6 +1566,84 @@ static int basicUnitTests(U32 seed, double compressibility)
     CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : ZSTD_lazy compress with hashLog = 29 and searchLog = 4 : ", testNb++);
+    if (MEM_64bits()) {
+        ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
+        ZSTD_inBuffer in = { CNBuffer, CNBufferSize, 0 };
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_strategy, ZSTD_lazy));
+        /* Force enable the row based match finder */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_searchLog, 4));
+        /* Set windowLog to 29 so the hashLog doesn't get sized down */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, 29));
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_hashLog, 29));
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
+        /* Compress with continue first so the hashLog doesn't get sized down */
+        CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_continue));
+        CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_end));
+        cSize = out.pos;
+        CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : Test offset == windowSize : ", testNb++);
+    {
+        int windowLog;
+        int const kMaxWindowLog = bigTests ? 29 : 26;
+        size_t const kNbSequences = 10000;
+        size_t const kMaxSrcSize = (1u << kMaxWindowLog) + 10 * kNbSequences;
+        char* src = calloc(kMaxSrcSize, 1);
+        ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
+        for (windowLog = ZSTD_WINDOWLOG_MIN; windowLog <= kMaxWindowLog; ++windowLog) {
+            size_t const srcSize = ((size_t)1 << windowLog) + 10 * (kNbSequences - 1);
+
+            sequences[0].offset = 32;
+            sequences[0].litLength = 32;
+            sequences[0].matchLength = (1u << windowLog) - 32;
+            sequences[0].rep = 0;
+            {
+                size_t i;
+                for (i = 1; i < kNbSequences; ++i) {
+                    sequences[i].offset = (1u << windowLog) - (FUZ_rand(&seed) % 8);
+                    sequences[i].litLength = FUZ_rand(&seed) & 7;
+                    sequences[i].matchLength = 10 - sequences[i].litLength;
+                    sequences[i].rep = 0;
+                }
+            }
+
+            CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_minMatch, 3));
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, 1));
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, windowLog));
+            assert(srcSize <= kMaxSrcSize);
+            cSize = ZSTD_compressSequences(zc, compressedBuffer, compressedBufferSize, sequences, kNbSequences, src, srcSize);
+            CHECK_Z(cSize);
+            CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
+            CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, windowLog))
+            {
+                ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
+                size_t decompressedBytes = 0;
+                for (;;) {
+                    ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
+                    size_t const ret = ZSTD_decompressStream(zd, &out, &in);
+                    CHECK_Z(ret);
+                    CHECK(decompressedBytes + out.pos > srcSize, "Output too large");
+                    CHECK(memcmp(out.dst, src + decompressedBytes, out.pos), "Corrupted");
+                    decompressedBytes += out.pos;
+                    if (ret == 0) {
+                        break;
+                    }
+                }
+                CHECK(decompressedBytes != srcSize, "Output wrong size");
+            }
+        }
+        free(sequences);
+        free(src);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     /* Overlen overwriting window data bug */
     DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
     {   /* This test has a window size of 1024 bytes and consists of 3 blocks:
@@ -1777,6 +1856,445 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : External matchfinder API: ", testNb++);
+    {
+        size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
+        BYTE* const dstBuf = (BYTE*)malloc(ZSTD_compressBound(dstBufSize));
+        size_t const checkBufSize = CNBufferSize;
+        BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
+        int enableFallback;
+        EMF_testCase externalMatchState;
+
+        CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
+
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+
+        /* Reference external matchfinder outside the test loop to
+         * check that the reference is preserved across compressions */
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+
+        for (enableFallback = 0; enableFallback <= 1; enableFallback++) {
+            size_t testCaseId;
+            size_t const numTestCases = 9;
+
+            EMF_testCase const testCases[] = {
+                EMF_ONE_BIG_SEQ,
+                EMF_LOTS_OF_SEQS,
+                EMF_ZERO_SEQS,
+                EMF_BIG_ERROR,
+                EMF_SMALL_ERROR,
+                EMF_INVALID_OFFSET,
+                EMF_INVALID_MATCHLEN,
+                EMF_INVALID_LITLEN,
+                EMF_INVALID_LAST_LITS
+            };
+
+            ZSTD_ErrorCode const errorCodes[] = {
+                ZSTD_error_no_error,
+                ZSTD_error_no_error,
+                ZSTD_error_externalMatchFinder_failed,
+                ZSTD_error_externalMatchFinder_failed,
+                ZSTD_error_externalMatchFinder_failed,
+                ZSTD_error_externalSequences_invalid,
+                ZSTD_error_externalSequences_invalid,
+                ZSTD_error_externalSequences_invalid,
+                ZSTD_error_externalSequences_invalid
+            };
+
+            for (testCaseId = 0; testCaseId < numTestCases; testCaseId++) {
+                size_t res;
+
+                int const compressionShouldSucceed = (
+                    (errorCodes[testCaseId] == ZSTD_error_no_error) ||
+                    (enableFallback && errorCodes[testCaseId] == ZSTD_error_externalMatchFinder_failed)
+                );
+
+                int const testWithSequenceValidation = (
+                    testCases[testCaseId] == EMF_INVALID_OFFSET
+                );
+
+                externalMatchState = testCases[testCaseId];
+
+                ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
+                CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, testWithSequenceValidation));
+                CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
+                res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+
+                if (compressionShouldSucceed) {
+                    CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
+                    CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
+                    CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
+                } else {
+                    CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+                    CHECK(
+                        ZSTD_getErrorCode(res) != errorCodes[testCaseId],
+                        "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+                    );
+                }
+            }
+
+            /* Test compression with external matchfinder + empty src buffer */
+            {
+                size_t res;
+                externalMatchState = EMF_ZERO_SEQS;
+                ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
+                CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
+                res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, 0);
+                CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
+                CHECK(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res) != 0, "EMF: Empty src round trip failed!");
+            }
+        }
+
+        /* Test that reset clears the external matchfinder */
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+        externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+        CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
+
+        /* Test that registering mFinder == NULL clears the external matchfinder */
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+        externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+        ZSTD_registerExternalMatchFinder(zc, NULL, NULL); /* clear the external matchfinder */
+        CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
+
+        /* Test that external matchfinder doesn't interact with older APIs */
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+        externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+        CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
+
+        /* Test that compression returns the correct error with LDM */
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+        {
+            size_t res;
+            ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
+            res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+            CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+            CHECK(
+                ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
+                "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+            );
+        }
+
+#ifdef ZSTD_MULTITHREAD
+        /* Test that compression returns the correct error with nbWorkers > 0 */
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+        {
+            size_t res;
+            ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
+            res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+            CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+            CHECK(
+                ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
+                "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+            );
+        }
+#endif
+
+        free(dstBuf);
+        free(checkBuf);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+
+    /* Test maxBlockSize cctx param functionality */
+    DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
+    {
+        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+
+        /* Quick test to make sure maxBlockSize bounds are enforced */
+        assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
+        assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
+
+        /* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/
+        {
+            size_t srcSize = 2 << 10;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst1 = compressedBuffer;
+            void* const dst2 = (BYTE*)compressedBuffer + dstSize;
+            size_t size1, size2;
+            void* const checkBuf = malloc(srcSize);
+            memset(src, 'x', srcSize);
+
+            /* maxBlockSize = 1KB */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            /* maxBlockSize = 3KB */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 - size2 == 4); /* We add another RLE block with header + character */
+            assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */
+
+            /* maxBlockSize = 1KB, windowLog = 10 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            /* maxBlockSize = 3KB, windowLog = 10 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 == size2);
+            assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
+
+            free(checkBuf);
+        }
+
+        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+        /* Test maxBlockSize = 0 is valid */
+        {   size_t srcSize = 256 << 10;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst1 = compressedBuffer;
+            void* const dst2 = (BYTE*)compressedBuffer + dstSize;
+            size_t size1, size2;
+            void* const checkBuf = malloc(srcSize);
+
+            /* maxBlockSize = 0 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            /* maxBlockSize = ZSTD_BLOCKSIZE_MAX */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 == size2);
+            assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
+            free(checkBuf);
+        }
+        ZSTD_freeCCtx(cctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+    /* Test Sequence Validation */
+    DISPLAYLEVEL(3, "test%3i : Testing sequence validation: ", testNb++);
+    {
+        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+
+        /* Test minMatch >= 4, matchLength < 4 */
+        {
+            size_t srcSize = 11;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst = compressedBuffer;
+            size_t const kNbSequences = 4;
+            ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
+
+            memset(src, 'x', srcSize);
+
+            sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
+            sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[3] = (ZSTD_Sequence) {0, 1, 0, 0};
+
+            /* Test with sequence validation */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
+
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+            /* Test without sequence validation */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
+
+            free(sequences);
+        }
+
+        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+
+        /* Test with no block delim */
+        {
+            size_t srcSize = 4;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst = compressedBuffer;
+            size_t const kNbSequences = 1;
+            ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
+            void* const checkBuf = malloc(srcSize);
+
+            memset(src, 'x', srcSize);
+
+            sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
+
+            /* Test with sequence validation */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 3));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(ZSTD_isError(cSize), "Should not throw an error");
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst, cSize));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            free(sequences);
+            free(checkBuf);
+        }
+
+        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+        { /* Test case with two additional sequences */
+            size_t srcSize = 19;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst = compressedBuffer;
+            size_t const kNbSequences = 7;
+            ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
+
+            memset(src, 'x', srcSize);
+
+            sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
+            sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[3] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[4] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[5] = (ZSTD_Sequence) {1, 0, 3, 0};
+            sequences[6] = (ZSTD_Sequence) {0, 0, 0, 0};
+
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
+
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+            /* Test without sequence validation */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
+            CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
+
+            free(sequences);
+        }
+        ZSTD_freeCCtx(cctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+
+    DISPLAYLEVEL(3, "test%3i : Testing large offset with small window size: ", testNb++);
+    {
+        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+        ZSTD_DCtx* dctx = ZSTD_createDCtx();
+
+        /* Test large offset, small window size*/
+        {
+            size_t srcSize = 21;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst = compressedBuffer;
+            size_t const kNbSequences = 4;
+            ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
+            void* const checkBuf = malloc(srcSize);
+            const size_t largeDictSize = 1 << 25;
+            ZSTD_CDict* cdict = NULL;
+            ZSTD_DDict* ddict = NULL;
+
+            /* Generate large dictionary */
+            void* dictBuffer = calloc(largeDictSize, 1);
+            ZSTD_compressionParameters cParams = ZSTD_getCParams(1, srcSize, largeDictSize);
+            cParams.minMatch = ZSTD_MINMATCH_MIN;
+            cParams.hashLog = ZSTD_HASHLOG_MIN;
+            cParams.chainLog = ZSTD_CHAINLOG_MIN;
+
+            cdict = ZSTD_createCDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
+            ddict = ZSTD_createDDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
+
+            ZSTD_CCtx_refCDict(cctx, cdict);
+            ZSTD_DCtx_refDDict(dctx, ddict);
+
+            sequences[0] = (ZSTD_Sequence) {3, 3, 3, 0};
+            sequences[1] = (ZSTD_Sequence) {1 << 25, 0, 3, 0};
+            sequences[2] = (ZSTD_Sequence) {1 << 25, 0, 9, 0};
+            sequences[3] = (ZSTD_Sequence) {3, 0, 3, 0};
+
+            cSize = ZSTD_compressSequences(cctx, dst, dstSize,
+                                   sequences, kNbSequences,
+                                   src, srcSize);
+
+            CHECK(ZSTD_isError(cSize), "Should not throw an error");
+
+            {
+                size_t dSize = ZSTD_decompressDCtx(dctx, checkBuf, srcSize, dst, cSize);
+                CHECK(ZSTD_isError(dSize), "Should not throw an error");
+                CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+            }
+
+            free(sequences);
+            free(checkBuf);
+            free(dictBuffer);
+            ZSTD_freeCDict(cdict);
+            ZSTD_freeDDict(ddict);
+        }
+        ZSTD_freeCCtx(cctx);
+        ZSTD_freeDCtx(dctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
 _end:
     FUZ_freeDictionary(dictionary);
     ZSTD_freeCStream(zc);
@@ -2444,6 +2962,9 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
         CHECK(badParameters(zc, savedParams), "CCtx params are wrong");
 
         /* multi - fragments decompression test */
+        if (FUZ_rand(&lseed) & 1) {
+            CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
+        }
         if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
             DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", (void const*)dict);
             CHECK_Z( ZSTD_resetDStream(zd) );
@@ -2452,6 +2973,9 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
                 DISPLAYLEVEL(5, "using dictionary of size %zu \n", dictSize);
             CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
         }
+        if (FUZ_rand(&lseed) & 1) {
+            CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
+        }
         {   size_t decompressionResult = 1;
             ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
             ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
@@ -2493,7 +3017,14 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
         }   }
 
         /* try decompression on noisy data */
-        CHECK_Z( ZSTD_initDStream(zd_noise) );   /* note : no dictionary */
+        if (FUZ_rand(&lseed) & 1) {
+            CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_and_parameters));
+        } else {
+            CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_only));
+        }
+        if (FUZ_rand(&lseed) & 1) {
+            CHECK_Z(ZSTD_DCtx_setParameter(zd_noise, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
+        }
         {   ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
             ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
             while (outBuff.pos < dstBufferSize) {
@@ -2576,6 +3107,7 @@ int main(int argc, const char** argv)
 
             if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
             if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
+            if (!strcmp(argument, "--big-tests")) { bigTests=1; continue; }
 
             argument++;
             while (*argument!=0) {
@@ -2677,7 +3209,7 @@ int main(int argc, const char** argv)
     if (nbTests<=0) nbTests=1;
 
     if (testNb==0) {
-        result = basicUnitTests(0, ((double)proba) / 100);  /* constant seed for predictability */
+        result = basicUnitTests(0, ((double)proba) / 100, bigTests);  /* constant seed for predictability */
     }
 
     if (!result) {
diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md
index f1cc88c52..311e1db7a 100644
--- a/zlibWrapper/README.md
+++ b/zlibWrapper/README.md
@@ -1,7 +1,7 @@
 Zstandard wrapper for zlib
 ================================
 
-The main objective of creating a zstd wrapper for [zlib](http://zlib.net/) is to allow a quick and smooth transition to zstd for projects already using zlib.
+The main objective of creating a zstd wrapper for [zlib](https://zlib.net/) is to allow a quick and smooth transition to zstd for projects already using zlib.
 
 #### Required files
 
@@ -43,7 +43,7 @@ This behavior can be changed using `ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB)
 
 
 #### Example
-We have taken the file `test/example.c` from [the zlib library distribution](http://zlib.net/) and copied it to [zlibWrapper/examples/example.c](examples/example.c).
+We have taken the file `test/example.c` from [the zlib library distribution](https://zlib.net/) and copied it to [zlibWrapper/examples/example.c](examples/example.c).
 After compilation and execution it shows the following results: 
 ```
 zlib version 1.2.8 = 0x1280, compile flags = 0x65
diff --git a/zlibWrapper/examples/minigzip.c b/zlibWrapper/examples/minigzip.c
index fc9d263ca..717a94df9 100644
--- a/zlibWrapper/examples/minigzip.c
+++ b/zlibWrapper/examples/minigzip.c
@@ -3,7 +3,7 @@
 
 /* minigzip.c -- simulate gzip using the zlib compression library
  * Copyright (C) 1995-2006, 2010, 2011 Jean-loup Gailly.
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 /*
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index 8673ca348..3bc9a1ab0 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -388,7 +388,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
             markNb = (markNb+1) % NB_MARKS;
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
                     marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio,
-                    (double)srcSize / fastestC );
+                    (double)srcSize / (double)fastestC );
 
             (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
 #if 1
@@ -428,8 +428,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                     ZSTD_DStream* zbd = ZSTD_createDStream();
                     size_t rSize;
                     if (zbd == NULL) EXM_THROW(1, "ZSTD_createDStream() allocation failure");
-                    rSize = ZSTD_initDStream_usingDict(zbd, dictBuffer, dictBufferSize);
-                    if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initDStream() failed : %s", ZSTD_getErrorName(rSize));
+                    rSize = ZSTD_DCtx_reset(zbd, ZSTD_reset_session_only);
+                    if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_DCtx_reset() failed : %s", ZSTD_getErrorName(rSize));
+                    rSize = ZSTD_DCtx_loadDictionary(zbd, dictBuffer, dictBufferSize);
+                    if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_DCtx_loadDictionary() failed : %s", ZSTD_getErrorName(rSize));
                     do {
                         U32 blockNb;
                         for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
                     marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio,
-                    (double)srcSize / fastestC,
-                    (double)srcSize / fastestD );
+                    (double)srcSize / (double)fastestC,
+                    (double)srcSize / (double)fastestD );
 
             /* CRC Checking */
             {   U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
@@ -558,8 +560,8 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
         }   /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
 
         if (g_displayLevel == 1) {
-            double cSpeed = (double)srcSize / fastestC;
-            double dSpeed = (double)srcSize / fastestD;
+            double cSpeed = (double)srcSize / (double)fastestC;
+            double dSpeed = (double)srcSize / (double)fastestD;
             if (g_additionalParam)
                 DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
             else
diff --git a/zlibWrapper/gzclose.c b/zlibWrapper/gzclose.c
index 25d3789b1..ba43b8c5e 100644
--- a/zlibWrapper/gzclose.c
+++ b/zlibWrapper/gzclose.c
@@ -3,7 +3,7 @@
 
 /* gzclose.c -- zlib gzclose() function
  * Copyright (C) 2004, 2010 Mark Adler
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"
diff --git a/zlibWrapper/gzcompatibility.h b/zlibWrapper/gzcompatibility.h
index 3a46127b8..9d11b984f 100644
--- a/zlibWrapper/gzcompatibility.h
+++ b/zlibWrapper/gzcompatibility.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/zlibWrapper/gzguts.h b/zlibWrapper/gzguts.h
index 83db17c19..70a609d72 100644
--- a/zlibWrapper/gzguts.h
+++ b/zlibWrapper/gzguts.h
@@ -4,7 +4,7 @@
 
 /* gzguts.h -- zlib internal header definitions for gz* operations
  * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 #ifdef _LARGEFILE64_SOURCE
diff --git a/zlibWrapper/gzlib.c b/zlibWrapper/gzlib.c
index e2d2a31af..eea480a74 100644
--- a/zlibWrapper/gzlib.c
+++ b/zlibWrapper/gzlib.c
@@ -3,7 +3,7 @@
 
 /* gzlib.c -- zlib functions common to reading and writing gzip files
  * Copyright (C) 2004-2017 Mark Adler
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"
diff --git a/zlibWrapper/gzread.c b/zlibWrapper/gzread.c
index 18955de5f..584fad1ea 100644
--- a/zlibWrapper/gzread.c
+++ b/zlibWrapper/gzread.c
@@ -3,7 +3,7 @@
 
  /* gzread.c -- zlib functions for reading gzip files
  * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"
diff --git a/zlibWrapper/gzwrite.c b/zlibWrapper/gzwrite.c
index 277388f7f..ccd4f71f1 100644
--- a/zlibWrapper/gzwrite.c
+++ b/zlibWrapper/gzwrite.c
@@ -3,7 +3,7 @@
 
  /* gzwrite.c -- zlib functions for writing gzip files
  * Copyright (C) 2004-2017 Mark Adler
- * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
+ * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
  */
 
 #include 
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index 386c0f0ac..479ddd4e0 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -706,8 +706,10 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary _Z_OF((z_streamp strm,
 
     {   ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
         if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
-        { size_t const initErr = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
-          if (ZSTD_isError(initErr)) return ZWRAPD_finishWithError(zwd, strm, 0); }
+        { size_t const resetErr = ZSTD_DCtx_reset(zwd->zbd, ZSTD_reset_session_only);
+          if (ZSTD_isError(resetErr)) return ZWRAPD_finishWithError(zwd, strm, 0); }
+        { size_t const loadErr = ZSTD_DCtx_loadDictionary(zwd->zbd, dictionary, dictLength);
+          if (ZSTD_isError(loadErr)) return ZWRAPD_finishWithError(zwd, strm, 0); }
         zwd->decompState = ZWRAP_useReset;
 
         if (zwd->totalInBytes == ZSTD_HEADERSIZE) {
diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h
index c39cf6379..230bf8411 100644
--- a/zlibWrapper/zstd_zlibwrapper.h
+++ b/zlibWrapper/zstd_zlibwrapper.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the