krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-07 09:26:03 +02:00
Code Issues Releases 1 Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into mingw

Browse Source 
# Conflicts:
#	programs/platform.h
This commit is contained in:
Przemyslaw Skibinski 2017-02-10 13:38:54 +01:00
commit 456330948a
14 changed files with 54 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/common/zstd_common.c
View File

@ -41,7 +41,7 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
/*! ZSTD_getErrorString() : /*! ZSTD_getErrorString() :
* provides error code string from enum */ * provides error code string from enum */
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); } const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
/*=************************************************************** /*=**************************************************************

4
lib/deprecated/zbuff.h
View File

@ -42,7 +42,9 @@ extern "C" {
#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS #ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */ # define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */
#else #else
# if (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__) # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API
# elif (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message))) # define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3) # elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated)) # define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated))

2
lib/dictBuilder/zdict.h
View File

@ -174,7 +174,7 @@ ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBuffer
#else #else
# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZDICT_DEPRECATED(message) ZDICTLIB_API [[deprecated(message)]] # define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
# elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__) # elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__)
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message))) # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
# elif (ZDICT_GCC_VERSION >= 301) # elif (ZDICT_GCC_VERSION >= 301)

1
lib/dll/example/build_package.bat
View File

@ -4,6 +4,7 @@ COPY tests\fullbench.c bin\example\
COPY programs\datagen.c bin\example\ COPY programs\datagen.c bin\example\
COPY programs\datagen.h bin\example\ COPY programs\datagen.h bin\example\
COPY programs\util.h bin\example\ COPY programs\util.h bin\example\
COPY programs\platform.h bin\example\
COPY lib\common\mem.h bin\example\ COPY lib\common\mem.h bin\example\
COPY lib\common\zstd_errors.h bin\example\ COPY lib\common\zstd_errors.h bin\example\
COPY lib\common\zstd_internal.h bin\example\ COPY lib\common\zstd_internal.h bin\example\

5
programs/Makefile
View File

@ -67,8 +67,9 @@ endif
# zlib detection # zlib detection
VOID = /dev/null VOID = /dev/null
HAVE_ZLIB := $(shell echo "\#include <zlib.h>\nint main(){}" | $(CC) -o $(VOID) -x c - -lz 2> $(VOID) && echo 1 || echo 0) HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0)
ifeq ($(HAVE_ZLIB), 1) ifeq ($(HAVE_ZLIB), 1)
TEMP := $(shell rm have_zlib$(EXT))
ZLIBCPP = -DZSTD_GZDECOMPRESS ZLIBCPP = -DZSTD_GZDECOMPRESS
ZLIBLD = -lz ZLIBLD = -lz
endif endif
@ -152,7 +153,7 @@ clean:
@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda @$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ @$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
*.gcda default.profraw have_zlib *.gcda default.profraw have_zlib$(EXT)
@echo Cleaning completed @echo Cleaning completed
clean_decomp_o: clean_decomp_o:

23
programs/bench.c
View File

@ -449,7 +449,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast, const char* displayName, int cLevel, int cLevelLast,
const size_t* fileSizes, unsigned nbFiles, const size_t* fileSizes, unsigned nbFiles,
const void* dictBuffer, size_t dictBufferSize, const void* dictBuffer, size_t dictBufferSize,
ZSTD_compressionParameters *compressionParams) ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
{ {
int l; int l;
@ -457,7 +457,10 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1; if (pch) displayName = pch+1;
SET_HIGH_PRIORITY; if (setRealTimePrio) {
DISPLAYLEVEL(2, "Note : switching to a real-time priority \n");
SET_REALTIME_PRIORITY;
}
if (g_displayLevel == 1 && !g_additionalParam) if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
@ -505,8 +508,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
if (totalSize == 0) EXM_THROW(12, "no data to bench"); if (totalSize == 0) EXM_THROW(12, "no data to bench");
} }
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel,
int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams) int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
{ {
void* srcBuffer; void* srcBuffer;
size_t benchedSize; size_t benchedSize;
@ -545,7 +548,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
BMK_benchCLevel(srcBuffer, benchedSize, BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast, displayName, cLevel, cLevelLast,
fileSizes, nbFiles, fileSizes, nbFiles,
dictBuffer, dictBufferSize, compressionParams); dictBuffer, dictBufferSize, compressionParams, setRealTimePrio);
} }
/* clean up */ /* clean up */
@ -555,7 +558,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
} }
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams) static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
{ {
char name[20] = {0}; char name[20] = {0};
size_t benchedSize = 10000000; size_t benchedSize = 10000000;
@ -569,7 +572,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
/* Bench */ /* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams); BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio);
/* clean up */ /* clean up */
free(srcBuffer); free(srcBuffer);
@ -577,7 +580,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams) int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
{ {
double const compressibility = (double)g_compressibilityDefault / 100; double const compressibility = (double)g_compressibilityDefault / 100;
@ -587,8 +590,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (nbFiles == 0) if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams); BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio);
else else
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams); BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio);
return 0; return 0;
} }

2
programs/bench.h
View File

@ -16,7 +16,7 @@
#include "zstd.h" /* ZSTD_compressionParameters */ #include "zstd.h" /* ZSTD_compressionParameters */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName, int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams); int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio);
/* Set Parameters */ /* Set Parameters */
void BMK_setNbSeconds(unsigned nbLoops); void BMK_setNbSeconds(unsigned nbLoops);

8
programs/platform.h
View File

@ -1,6 +1,6 @@
/** /**
* platform.h - compiler and OS detection * platform.h - compiler and OS detection
* *
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved. * All rights reserved.
* *
@ -23,7 +23,7 @@ extern "C" {
****************************************/ ****************************************/
#if defined(_MSC_VER) #if defined(_MSC_VER)
# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ # define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */ # define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ # if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
# endif # endif
@ -77,7 +77,9 @@ extern "C" {
# define PLATFORM_POSIX_VERSION 200112L # define PLATFORM_POSIX_VERSION 200112L
# else # else
# if defined(__linux__) || defined(__linux) # if defined(__linux__) || defined(__linux)
# define _POSIX_C_SOURCE 200112L /* use feature test macro */ # ifndef _POSIX_C_SOURCE
# define _POSIX_C_SOURCE 200112L /* use feature test macro */
# endif
# endif # endif
# include <unistd.h> /* declares _POSIX_VERSION */ # include <unistd.h> /* declares _POSIX_VERSION */
# if defined(_POSIX_VERSION) /* POSIX compliant */ # if defined(_POSIX_VERSION) /* POSIX compliant */

8
programs/util.h
View File

@ -44,7 +44,7 @@ extern "C" {
******************************************/ ******************************************/
#if defined(_WIN32) #if defined(_WIN32)
# include <windows.h> # include <windows.h>
# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
# define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleep(s) Sleep(1000*s)
# define UTIL_sleepMilli(milli) Sleep(milli) # define UTIL_sleepMilli(milli) Sleep(milli)
#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ #elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
@ -52,9 +52,9 @@ extern "C" {
# include <sys/resource.h> /* setpriority */ # include <sys/resource.h> /* setpriority */
# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ # include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
# if defined(PRIO_PROCESS) # if defined(PRIO_PROCESS)
# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) # define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
# else # else
# define SET_HIGH_PRIORITY /* disabled */ # define SET_REALTIME_PRIORITY /* disabled */
# endif # endif
# define UTIL_sleep(s) sleep(s) # define UTIL_sleep(s) sleep(s)
# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ # if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */
@ -63,7 +63,7 @@ extern "C" {
# define UTIL_sleepMilli(milli) /* disabled */ # define UTIL_sleepMilli(milli) /* disabled */
# endif # endif
#else #else
# define SET_HIGH_PRIORITY /* disabled */ # define SET_REALTIME_PRIORITY /* disabled */
# define UTIL_sleep(s) /* disabled */ # define UTIL_sleep(s) /* disabled */
# define UTIL_sleepMilli(milli) /* disabled */ # define UTIL_sleepMilli(milli) /* disabled */
#endif #endif

3
programs/zstd.1
View File

@ -289,7 +289,8 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1
.TP .TP
.B \-B# .B \-B#
cut file into independent blocks of size # (default: no block) cut file into independent blocks of size # (default: no block)
.B \--priority=rt
set process priority to real-time
.SH ADVANCED COMPRESSION OPTIONS .SH ADVANCED COMPRESSION OPTIONS
.TP .TP

7
programs/zstdcli.c
View File

@ -148,6 +148,7 @@ static int usage_advanced(const char* programName)
DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n");
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
DISPLAY( "--priority=rt : set process priority to real-time\n");
#endif #endif
return 0; return 0;
} }
@ -267,7 +268,8 @@ int main(int argCount, const char* argv[])
nextArgumentsAreFiles=0, nextArgumentsAreFiles=0,
ultra=0, ultra=0,
lastCommand = 0, lastCommand = 0,
nbThreads = 1; nbThreads = 1,
setRealTimePrio = 0;
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0; size_t blockSize = 0;
zstd_operation_mode operation = zom_compress; zstd_operation_mode operation = zom_compress;
@ -356,6 +358,7 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
/* long commands with arguments */ /* long commands with arguments */
#ifndef ZSTD_NODICT #ifndef ZSTD_NODICT
@ -574,7 +577,7 @@ int main(int argCount, const char* argv[])
BMK_setBlockSize(blockSize); BMK_setBlockSize(blockSize);
BMK_setNbThreads(nbThreads); BMK_setNbThreads(nbThreads);
BMK_setNbSeconds(bench_nbSeconds); BMK_setNbSeconds(bench_nbSeconds);
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams); BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio);
#endif #endif
(void)bench_nbSeconds; (void)bench_nbSeconds;
goto _end; goto _end;

5
tests/Makefile
View File

@ -225,7 +225,10 @@ zstd-playTests: datagen
file $(ZSTD) file $(ZSTD)
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-pool test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries
ifneq ($(QEMU_SYS),qemu-ppc64-static)
test: test-pool
endif
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32

10
tests/fuzzer.c
View File

@ -505,6 +505,16 @@ static int basicUnitTests(U32 seed, double compressibility)
if (r != _3BYTESTESTLENGTH) goto _output_error; } if (r != _3BYTESTESTLENGTH) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* error string tests */
DISPLAYLEVEL(4, "test%3i : testing ZSTD error code strings : ", testNb++);
if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
_end: _end:
free(CNBuffer); free(CNBuffer);
free(compressedBuffer); free(compressedBuffer);

2
zlibWrapper/examples/zwrapbench.c
View File

@ -591,7 +591,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1; if (pch) displayName = pch+1;
SET_HIGH_PRIORITY; SET_REALTIME_PRIORITY;
if (g_displayLevel == 1 && !g_additionalParam) if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));