diff --git a/Makefile b/Makefile index dc2ac4ce0..54652665b 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) HOST_OS = POSIX -CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON +CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON .PHONY: list list: diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index 9ce52ed21..bf6890818 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -8,7 +8,7 @@ # ################################################################ PROJECT(zstd) -CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9) SET(ZSTD_SOURCE_DIR "${CMAKE_SOURCE_DIR}/../..") #----------------------------------------------------------------------------- diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt index b4fbe1689..71def02cd 100644 --- a/build/cmake/contrib/pzstd/CMakeLists.txt +++ b/build/cmake/contrib/pzstd/CMakeLists.txt @@ -21,8 +21,8 @@ SET(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd) INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR}) ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp) -SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_DEFINITIONS "NDEBUG") -SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_OPTIONS "-Wno-shadow") +SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG") +SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow") SET(THREADS_PREFER_PTHREAD_FLAG ON) FIND_PACKAGE(Threads REQUIRED) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 7a345bf56..5e945ff12 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -97,9 +97,9 @@ ENDIF (ZSTD_BUILD_STATIC) # Add specific compile definitions for MSVC project IF (MSVC) - SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS") + SET_PROPERTY(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS") IF (ZSTD_BUILD_STATIC) - SET_TARGET_PROPERTIES(libzstd_static PROPERTIES COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS") + SET_PROPERTY(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS") ENDIF (ZSTD_BUILD_STATIC) ENDIF (MSVC) @@ -130,8 +130,20 @@ IF (ZSTD_BUILD_STATIC) ENDIF (ZSTD_BUILD_STATIC) IF (UNIX) + # pkg-config + SET(PREFIX "${CMAKE_INSTALL_PREFIX}") + SET(LIBDIR "${CMAKE_INSTALL_PREFIX}/lib") + SET(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include") + SET(VERSION "${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") + ADD_CUSTOM_TARGET(libzstd.pc ALL + ${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc" + -DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}" + -P "${CMAKE_SOURCE_DIR}/lib/pkgconfig.cmake" + COMMENT "Creating pkg-config file") + # install target INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/deprecated/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION "include") + INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "share/pkgconfig") INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION "lib") IF (ZSTD_BUILD_STATIC) INSTALL(TARGETS libzstd_static ARCHIVE DESTINATION "lib") diff --git a/build/cmake/lib/pkgconfig.cmake b/build/cmake/lib/pkgconfig.cmake new file mode 100644 index 000000000..5434ff75c --- /dev/null +++ b/build/cmake/lib/pkgconfig.cmake @@ -0,0 +1 @@ +CONFIGURE_FILE("${IN}" "${OUT}" @ONLY) diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt index 588fea41e..cb6aa9218 100644 --- a/build/cmake/programs/CMakeLists.txt +++ b/build/cmake/programs/CMakeLists.txt @@ -31,27 +31,29 @@ ENDIF (MSVC) ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources}) TARGET_LINK_LIBRARIES(zstd libzstd_shared) -ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd zstdcat COMMENT "Creating zstdcat symlink") -ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd unzstd COMMENT "Creating unzstd symlink") +ADD_CUSTOM_TARGET(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink") +ADD_CUSTOM_TARGET(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink") INSTALL(TARGETS zstd RUNTIME DESTINATION "bin") INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "bin") INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "bin") IF (UNIX) - FILE(COPY ${PROGRAMS_DIR}/zstd.1 DESTINATION .) - ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 COMMENT "Creating zstdcat.1 symlink") - ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 COMMENT "Creating unzstd.1 symlink") + ADD_CUSTOM_TARGET(zstd.1 ALL + ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 . + COMMENT "Copying manpage zstd.1") + ADD_CUSTOM_TARGET(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink") + ADD_CUSTOM_TARGET(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink") INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "share/man/man1") INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "share/man/man1") INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "share/man/man1") ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c) TARGET_LINK_LIBRARIES(zstd-frugal libzstd_shared) - SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT") + SET_PROPERTY(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT") ENDIF (UNIX) IF (ZSTD_MULTITHREAD_SUPPORT) - SET_TARGET_PROPERTIES(zstd PROPERTIES COMPILE_DEFINITIONS "ZSTD_MULTITHREAD") + SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD") SET(THREADS_PREFER_PTHREAD_FLAG ON) FIND_PACKAGE(Threads REQUIRED) @@ -61,6 +63,33 @@ IF (ZSTD_MULTITHREAD_SUPPORT) MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads") ENDIF() - ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd zstdmt COMMENT "Creating zstdmt symlink") + ADD_CUSTOM_TARGET(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink") INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin") ENDIF (ZSTD_MULTITHREAD_SUPPORT) + +OPTION(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF) +OPTION(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF) + +IF (ZSTD_ZLIB_SUPPORT) + FIND_PACKAGE(ZLIB REQUIRED) + + IF (ZLIB_FOUND) + INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIRS}) + TARGET_LINK_LIBRARIES(zstd ${ZLIB_LIBRARIES}) + SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS") + ELSE () + MESSAGE(SEND_ERROR "zlib library is missing") + ENDIF () +ENDIF () + +IF (ZSTD_LZMA_SUPPORT) + FIND_PACKAGE(LibLZMA REQUIRED) + + IF (LIBLZMA_FOUND) + INCLUDE_DIRECTORIES(${LIBLZMA_INCLUDE_DIRS}) + TARGET_LINK_LIBRARIES(zstd ${LIBLZMA_LIBRARIES}) + SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS") + ELSE () + MESSAGE(SEND_ERROR "lzma library is missing") + ENDIF () +ENDIF () diff --git a/circle.yml b/circle.yml index 298569d14..218e33bfc 100644 --- a/circle.yml +++ b/circle.yml @@ -3,7 +3,7 @@ dependencies: - sudo dpkg --add-architecture i386 - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 + - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 zlib1g-dev liblzma-dev - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386 test: diff --git a/lib/Makefile b/lib/Makefile index 197fdeeea..d8d8e179d 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -71,6 +71,9 @@ libzstd.a: $(ZSTD_OBJ) @echo compiling static library @$(AR) $(ARFLAGS) $@ $^ +libzstd.a-mt: CPPFLAGS += -DZSTD_MULTHREAD +libzstd.a-mt: libzstd.a + $(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden $(LIBZSTD): $(ZSTD_FILES) @echo compiling dynamic library $(LIBVER) @@ -86,10 +89,17 @@ endif libzstd : $(LIBZSTD) +libzstd-mt : CPPFLAGS += -DZSTD_MULTITHREAD +libzstd-mt : libzstd + lib: libzstd.a libzstd -lib-release: DEBUGFLAGS := +lib-mt: CPPFLAGS += -DZSTD_MULTITHREAD +lib-mt: lib + +lib-release lib-release-mt: DEBUGFLAGS := lib-release: lib +lib-release-mt: lib-mt clean: @$(RM) -r *.dSYM # Mac OS-X specific diff --git a/lib/README.md b/lib/README.md index 3357e3d87..79b6fd500 100644 --- a/lib/README.md +++ b/lib/README.md @@ -22,6 +22,14 @@ Some additional API may be useful if you're looking into advanced features : They are not "stable", their definition may change in the future. Only static linking is allowed. +#### ZSTDMT API + +To enable multithreaded compression within the library, invoke `make lib-mt` target. +Prototypes are defined in header file `compress/zstdmt_compress.h`. +When linking a program that uses ZSTDMT API against libzstd.a on a POSIX system, +`-pthread` flag must be provided to the compiler and linker. +Note : ZSTDMT prototypes can still be used with a library built without multithread support, +but in this case, they will be single threaded only. #### Modular build diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 29dc0b773..891094e46 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -535,8 +535,10 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, { HUF_repeat repeat = zc->flagStaticHufTable; int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; - cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat) - : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat); + cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, + zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat) + : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, + zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat); if (repeat != HUF_repeat_none) { hType = set_repeat; } /* reused the existing table */ else { zc->flagStaticHufTable = HUF_repeat_check; } /* now have a table to reuse */ } @@ -881,7 +883,14 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctzll((U64)val) >> 3); # else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, + 0, 3, 1, 3, 1, 4, 2, 7, + 0, 2, 3, 6, 1, 5, 3, 5, + 1, 3, 4, 4, 2, 5, 6, 7, + 7, 0, 1, 2, 3, 3, 4, 6, + 2, 6, 5, 5, 3, 4, 5, 6, + 7, 1, 2, 4, 6, 4, 4, 5, + 7, 2, 6, 5, 7, 6, 7, 7 }; return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; # endif } else { /* 32 bits */ @@ -892,7 +901,10 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctz((U32)val) >> 3); # else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, + 3, 2, 2, 1, 3, 2, 0, 1, + 3, 3, 1, 2, 2, 2, 2, 0, + 3, 1, 2, 0, 1, 0, 1, 1 }; return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; # endif } diff --git a/programs/Makefile b/programs/Makefile index f3140a19c..4ab4dfd62 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -93,7 +93,7 @@ THREAD_MSG := ==> building with threading support THREAD_CPP := -DZSTD_MULTITHREAD THREAD_LD := -pthread else -THREAD_MSG := NO_THREAD_MSG +THREAD_MSG := $(NO_THREAD_MSG) endif # zlib detection @@ -161,7 +161,7 @@ zstd-nolegacy : clean_decomp_o zstd-nomt : THREAD_CPP := zstd-nomt : THREAD_LD := -zstd-nomt : THREAD_MSG := NO_THREAD_MSG +zstd-nomt : THREAD_MSG := $(NO_THREAD_MSG) zstd-nomt : zstd zstd-pgo : MOREFLAGS = -fprofile-generate