mirror of
https://github.com/facebook/zstd.git
synced 2025-03-07 01:10:04 +02:00
Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11
This commit is contained in:
Przemyslaw Skibinski
commit
7da2f7fc8e
9
NEWS
9
NEWS
@ -1,8 +1,13 @@
|
||||
v1.1.2
|
||||
Improved : faster decompression speed at ultra compression settings and in 32-bits mode
|
||||
cli : new : preserve file attributes, by Przemyslaw Skibinski
|
||||
cli : fixed : status displays total amount decoded when stream/file consists of multiple appended frames (like pzstd)
|
||||
cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
|
||||
cli : new : preserve file attributes
|
||||
cli : new : added zstdless and zstdgrep tools
|
||||
cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
|
||||
cli : fixed : zstdcat
|
||||
API : changed : zbuff prototypes now generate deprecation warnings
|
||||
API : changed : streaming decompression implicit reset on starting new frame
|
||||
Changed : reduced stack memory use
|
||||
|
||||
v1.1.1
|
||||
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
|
||||
|
||||
@ -358,13 +358,15 @@ typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* doubl
|
||||
|
||||
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
|
||||
|
||||
/* HUF_fillDTableX4Level2() :
|
||||
* `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
|
||||
static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed,
|
||||
const U32* rankValOrigin, const int minWeight,
|
||||
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,
|
||||
U32 nbBitsBaseline, U16 baseSeq)
|
||||
{
|
||||
HUF_DEltX4 DElt;
|
||||
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
|
||||
U32 rankVal[HUF_TABLELOG_MAX + 1];
|
||||
|
||||
/* get pre-calculated rankVal */
|
||||
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
|
||||
|
||||
@ -1959,7 +1959,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
switch(zds->stage)
|
||||
{
|
||||
case zdss_init :
|
||||
return ERROR(init_missing);
|
||||
ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
|
||||
/* fall-through */
|
||||
|
||||
case zdss_loadHeader :
|
||||
{ size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
|
||||
|
||||
@ -299,8 +299,8 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
|
||||
* If `output.pos < output.size`, decoder has flushed everything it could.
|
||||
* @return : 0 when a frame is completely decoded and fully flushed,
|
||||
* an error code, which can be tested using ZSTD_isError(),
|
||||
* any other value > 0, which means there is still some work to do to complete the frame.
|
||||
* The return value is a suggested next input size (just an hint, to help latency).
|
||||
* any other value > 0, which means there is still some decoding to do to complete current frame.
|
||||
* The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
|
||||
* *******************************************************************************/
|
||||
|
||||
/*===== Streaming decompression functions =====*/
|
||||
|
||||
@ -27,13 +27,13 @@ else
|
||||
ALIGN_LOOP =
|
||||
endif
|
||||
|
||||
CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
|
||||
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
|
||||
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
|
||||
-Wpointer-arith
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
|
||||
|
||||
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
|
||||
@ -82,7 +82,7 @@ zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
windres\generate_res.bat
|
||||
endif
|
||||
$(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS)
|
||||
$(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS)
|
||||
|
||||
|
||||
zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
|
||||
@ -91,7 +91,7 @@ zstd32 : $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTD_FILES) $(ZSTDLEGACY_FILE
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
windres\generate_res.bat
|
||||
endif
|
||||
$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
|
||||
$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
|
||||
|
||||
|
||||
zstd-nolegacy : clean_decomp_o
|
||||
@ -110,23 +110,23 @@ zstd-pgo : clean zstd
|
||||
$(MAKE) zstd MOREFLAGS=-fprofile-use
|
||||
|
||||
zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT)
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT)
|
||||
|
||||
zstd-small: clean_decomp_o
|
||||
ZSTD_LEGACY_SUPPORT=0 CFLAGS="-Os -s" $(MAKE) zstd-frugal
|
||||
|
||||
zstd-decompress-clean: $(ZSTDDECOMP_O) $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o zstd-decompress$(EXT)
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o zstd-decompress$(EXT)
|
||||
|
||||
zstd-decompress: clean_decomp_o
|
||||
ZSTD_LEGACY_SUPPORT=0 $(MAKE) zstd-decompress-clean
|
||||
|
||||
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
|
||||
|
||||
gzstd: clean_decomp_o
|
||||
ifeq ($(shell ld -lz 2>/dev/null && echo -n true),true)
|
||||
$(MAKE) zstd MOREFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz"
|
||||
CPPFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz" $(MAKE) zstd
|
||||
else
|
||||
$(MAKE) zstd
|
||||
endif
|
||||
@ -153,9 +153,11 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr
|
||||
install: zstd
|
||||
@echo Installing binaries
|
||||
@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
|
||||
@install -m 755 zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT)
|
||||
@ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat
|
||||
@ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd
|
||||
@install -m 755 zstd $(DESTDIR)$(BINDIR)/zstd
|
||||
@ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat
|
||||
@ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd
|
||||
@install -m 755 zstdless $(DESTDIR)$(BINDIR)/zstdless
|
||||
@install -m 755 zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
|
||||
@echo Installing man pages
|
||||
@install -m 644 zstd.1 $(DESTDIR)$(MANDIR)/zstd.1
|
||||
@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1
|
||||
@ -163,9 +165,11 @@ install: zstd
|
||||
@echo zstd installation completed
|
||||
|
||||
uninstall:
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstdgrep
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstdless
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstdcat
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/unzstd
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT)
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstd
|
||||
@$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1
|
||||
@$(RM) $(DESTDIR)$(MANDIR)/unzstd.1
|
||||
@$(RM) $(DESTDIR)$(MANDIR)/zstd.1
|
||||
|
||||
@ -7,7 +7,6 @@
|
||||
* of patent rights can be found in the PATENTS file in the same directory.
|
||||
*/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Compiler Options
|
||||
***************************************/
|
||||
@ -587,7 +586,7 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
|
||||
@return : size of decoded frame
|
||||
*/
|
||||
unsigned long long FIO_decompressFrame(dRess_t ress,
|
||||
FILE* foutput, FILE* finput, size_t alreadyLoaded,
|
||||
FILE* finput, size_t alreadyLoaded,
|
||||
U64 alreadyDecoded)
|
||||
{
|
||||
U64 frameSize = 0;
|
||||
@ -610,7 +609,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
|
||||
if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
|
||||
|
||||
/* Write block */
|
||||
storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips);
|
||||
storedSkips = FIO_fwriteSparse(ress.dstFile, ress.dstBuffer, outBuff.pos, storedSkips);
|
||||
frameSize += outBuff.pos;
|
||||
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) );
|
||||
|
||||
@ -623,7 +622,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
|
||||
if (readSize < toRead) EXM_THROW(39, "Read error : premature end");
|
||||
} }
|
||||
|
||||
FIO_fwriteSparseEnd(foutput, storedSkips);
|
||||
FIO_fwriteSparseEnd(ress.dstFile, storedSkips);
|
||||
|
||||
return frameSize;
|
||||
}
|
||||
@ -650,47 +649,45 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
static size_t FIO_decompressGzFile(dRess_t ress, size_t headBufSize, const char* srcFileName, FILE* srcFile)
|
||||
static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, const char* srcFileName, size_t alreadyLoaded)
|
||||
{
|
||||
int ret;
|
||||
unsigned char in[1];
|
||||
unsigned char* headBuf = (unsigned char*)ress.srcBuffer;
|
||||
size_t decompBytes, outFileSize = 0;
|
||||
unsigned long long outFileSize = 0;
|
||||
z_stream strm;
|
||||
|
||||
|
||||
strm.zalloc = Z_NULL;
|
||||
strm.zfree = Z_NULL;
|
||||
strm.opaque = Z_NULL;
|
||||
strm.next_in = 0;
|
||||
strm.avail_in = Z_NULL;
|
||||
if (inflateInit2(&strm, 15 + 16) != Z_OK) return 0;
|
||||
|
||||
if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0; /* see http://www.zlib.net/manual.html */
|
||||
|
||||
strm.next_out = ress.dstBuffer;
|
||||
strm.avail_out = ress.dstBufferSize;
|
||||
|
||||
for ( ; ; ) {
|
||||
if (headBufSize) {
|
||||
headBufSize--;
|
||||
unsigned char in[1];
|
||||
if (alreadyLoaded) {
|
||||
alreadyLoaded--;
|
||||
in[0] = *headBuf++;
|
||||
} else {
|
||||
if (fread(in, 1, 1, srcFile) == 0) break;
|
||||
}
|
||||
strm.next_in = in;
|
||||
strm.avail_in = 1;
|
||||
ret = inflate(&strm, Z_NO_FLUSH);
|
||||
if (ret == Z_STREAM_END) break;
|
||||
if (ret != Z_OK) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
|
||||
|
||||
decompBytes = ress.dstBufferSize - strm.avail_out;
|
||||
if (decompBytes) {
|
||||
if (fwrite(ress.dstBuffer, 1, decompBytes, ress.dstFile) != (size_t)decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
|
||||
outFileSize += decompBytes;
|
||||
strm.next_out = ress.dstBuffer;
|
||||
strm.avail_out = ress.dstBufferSize;
|
||||
{ int const ret = inflate(&strm, Z_NO_FLUSH);
|
||||
if (ret == Z_STREAM_END) break;
|
||||
if (ret != Z_OK) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
|
||||
}
|
||||
}
|
||||
|
||||
{ size_t const decompBytes = ress.dstBufferSize - strm.avail_out;
|
||||
if (decompBytes) {
|
||||
if (fwrite(ress.dstBuffer, 1, decompBytes, ress.dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
|
||||
outFileSize += decompBytes;
|
||||
strm.next_out = ress.dstBuffer;
|
||||
strm.avail_out = ress.dstBufferSize;
|
||||
} } }
|
||||
|
||||
inflateEnd(&strm);
|
||||
return outFileSize;
|
||||
@ -703,12 +700,11 @@ static size_t FIO_decompressGzFile(dRess_t ress, size_t headBufSize, const char*
|
||||
@return : 0 : OK
|
||||
1 : operation not started
|
||||
*/
|
||||
static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
|
||||
static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName)
|
||||
{
|
||||
unsigned long long filesize = 0;
|
||||
FILE* const dstFile = ress.dstFile;
|
||||
FILE* srcFile;
|
||||
unsigned readSomething = 0;
|
||||
unsigned long long filesize = 0;
|
||||
|
||||
if (UTIL_isDirectory(srcFileName)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
|
||||
@ -732,17 +728,17 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
|
||||
if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
|
||||
if (buf[0] == 31 && buf[1] == 139) { /* gz header */
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
size_t const result = FIO_decompressGzFile(ress, toRead, srcFileName, srcFile);
|
||||
unsigned long long const result = FIO_decompressGzFrame(ress, srcFile, srcFileName, toRead);
|
||||
if (result == 0) return 1;
|
||||
filesize += result;
|
||||
#else
|
||||
DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed -- ignored (zstd compiled without ZSTD_GZDECOMPRESS) \n", srcFileName);
|
||||
DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName);
|
||||
return 1;
|
||||
#endif
|
||||
} else {
|
||||
if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
|
||||
if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */
|
||||
unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
|
||||
if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
|
||||
unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
|
||||
if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */
|
||||
return result;
|
||||
} else {
|
||||
@ -750,7 +746,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
|
||||
fclose(srcFile);
|
||||
return 1;
|
||||
} }
|
||||
filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize);
|
||||
filesize += FIO_decompressFrame(ress, srcFile, toRead, filesize);
|
||||
}
|
||||
}
|
||||
|
||||
@ -771,7 +767,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
|
||||
1 : operation aborted (src not available, dst already taken, etc.)
|
||||
*/
|
||||
static int FIO_decompressDstFile(dRess_t ress,
|
||||
const char* dstFileName, const char* srcFileName)
|
||||
const char* dstFileName, const char* srcFileName)
|
||||
{
|
||||
int result;
|
||||
stat_t statbuf;
|
||||
@ -781,7 +777,7 @@ static int FIO_decompressDstFile(dRess_t ress,
|
||||
if (ress.dstFile==0) return 1;
|
||||
|
||||
if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1;
|
||||
result = FIO_decompressSrcFile(ress, srcFileName);
|
||||
result = FIO_decompressSrcFile(ress, dstFileName, srcFileName);
|
||||
|
||||
if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
|
||||
|
||||
@ -818,12 +814,12 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
|
||||
|
||||
if (suffix==NULL) EXM_THROW(70, "zstd: decompression: unknown dst"); /* should never happen */
|
||||
|
||||
if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) {
|
||||
if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) { /* special cases : -c or -t */
|
||||
unsigned u;
|
||||
ress.dstFile = FIO_openDstFile(suffix);
|
||||
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix);
|
||||
for (u=0; u<nbFiles; u++)
|
||||
missingFiles += FIO_decompressSrcFile(ress, srcNamesTable[u]);
|
||||
missingFiles += FIO_decompressSrcFile(ress, suffix, srcNamesTable[u]);
|
||||
if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close stdout");
|
||||
} else {
|
||||
size_t const suffixSize = strlen(suffix);
|
||||
|
||||
@ -18,7 +18,7 @@ extern "C" {
|
||||
/* *************************************
|
||||
* Special i/o constants
|
||||
**************************************/
|
||||
#define stdinmark "/*stdin*\\"
|
||||
#define stdinmark "/*stdin*\\"
|
||||
#define stdoutmark "/*stdout*\\"
|
||||
#ifdef _WIN32
|
||||
# define nulmark "nul"
|
||||
|
||||
@ -22,7 +22,7 @@ is equivalent to
|
||||
.br
|
||||
.B zstdcat
|
||||
is equivalent to
|
||||
.BR "zstd \-dc"
|
||||
.BR "zstd \-dcf"
|
||||
.br
|
||||
|
||||
.SH DESCRIPTION
|
||||
|
||||
@ -191,10 +191,15 @@ static unsigned readU32FromChar(const char** stringPtr)
|
||||
return result;
|
||||
}
|
||||
|
||||
/** longCommandWArg() :
|
||||
* check is *stringPtr is the same as longCommand.
|
||||
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
|
||||
* @return 0 and doesn't modify *stringPtr otherwise.
|
||||
*/
|
||||
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
{
|
||||
size_t const comSize = strlen(longCommand);
|
||||
unsigned const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||
if (result) *stringPtr += comSize;
|
||||
return result;
|
||||
}
|
||||
@ -213,7 +218,7 @@ int main(int argCount, const char* argv[])
|
||||
nextArgumentIsOutFileName=0,
|
||||
nextArgumentIsMaxDict=0,
|
||||
nextArgumentIsDictID=0,
|
||||
nextArgumentIsFile=0,
|
||||
nextArgumentsAreFiles=0,
|
||||
ultra=0,
|
||||
lastCommand = 0;
|
||||
zstd_operation_mode operation = zom_compress;
|
||||
@ -252,45 +257,15 @@ int main(int argCount, const char* argv[])
|
||||
|
||||
/* preset behaviors */
|
||||
if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress;
|
||||
if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
|
||||
if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; }
|
||||
|
||||
/* command switches */
|
||||
for (argNb=1; argNb<argCount; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
if (nextArgumentIsFile==0) {
|
||||
|
||||
/* long commands (--long-word) */
|
||||
if (!strcmp(argument, "--")) { nextArgumentIsFile=1; continue; } /* only file names allowed from now on */
|
||||
if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
|
||||
if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
|
||||
if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
|
||||
if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; }
|
||||
if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
|
||||
if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
|
||||
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
|
||||
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
|
||||
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; }
|
||||
if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
|
||||
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
|
||||
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
|
||||
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
|
||||
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
|
||||
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
|
||||
if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
|
||||
if (!strcmp(argument, "--train")) { operation=zom_train; outFileName=g_defaultDictName; continue; }
|
||||
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; }
|
||||
if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; }
|
||||
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
|
||||
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
|
||||
|
||||
/* long commands with arguments */
|
||||
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
|
||||
/* '-' means stdin/stdout */
|
||||
if (nextArgumentsAreFiles==0) {
|
||||
/* "-" means stdin/stdout */
|
||||
if (!strcmp(argument, "-")){
|
||||
if (!filenameIdx) {
|
||||
filenameIdx=1, filenameTable[0]=stdinmark;
|
||||
@ -301,8 +276,40 @@ int main(int argCount, const char* argv[])
|
||||
|
||||
/* Decode commands (note : aggregated commands are allowed) */
|
||||
if (argument[0]=='-') {
|
||||
argument++;
|
||||
|
||||
if (argument[1]=='-') {
|
||||
/* long commands (--long-word) */
|
||||
if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */
|
||||
if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
|
||||
if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
|
||||
if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
|
||||
if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; }
|
||||
if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
|
||||
if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
|
||||
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
|
||||
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
|
||||
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; }
|
||||
if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
|
||||
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
|
||||
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
|
||||
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
|
||||
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
|
||||
if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
|
||||
if (!strcmp(argument, "--train")) { operation=zom_train; outFileName=g_defaultDictName; continue; }
|
||||
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; }
|
||||
if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; }
|
||||
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
|
||||
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
|
||||
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
|
||||
|
||||
/* long commands with arguments */
|
||||
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
/* fall-through, will trigger bad_usage() later on */
|
||||
}
|
||||
|
||||
argument++;
|
||||
while (argument[0]!=0) {
|
||||
if (lastCommand) {
|
||||
DISPLAY("error : command must be followed by argument \n");
|
||||
|
||||
124
programs/zstdgrep
Executable file
124
programs/zstdgrep
Executable file
@ -0,0 +1,124 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Copyright (c) 2003 Thomas Klausner.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
||||
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
||||
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
grep=grep
|
||||
zcat=zstdcat
|
||||
|
||||
endofopts=0
|
||||
pattern_found=0
|
||||
grep_args=""
|
||||
hyphen=0
|
||||
silent=0
|
||||
|
||||
prg=$(basename $0)
|
||||
|
||||
# handle being called 'zegrep' or 'zfgrep'
|
||||
case ${prg} in
|
||||
*zegrep)
|
||||
grep_args="-E";;
|
||||
*zfgrep)
|
||||
grep_args="-F";;
|
||||
esac
|
||||
|
||||
# skip all options and pass them on to grep taking care of options
|
||||
# with arguments, and if -e was supplied
|
||||
|
||||
while [ $# -gt 0 -a ${endofopts} -eq 0 ]
|
||||
do
|
||||
case $1 in
|
||||
# from GNU grep-2.5.1 -- keep in sync!
|
||||
-[ABCDXdefm])
|
||||
if [ $# -lt 2 ]
|
||||
then
|
||||
echo "${prg}: missing argument for $1 flag" >&2
|
||||
exit 1
|
||||
fi
|
||||
case $1 in
|
||||
-e)
|
||||
pattern="$2"
|
||||
pattern_found=1
|
||||
shift 2
|
||||
break
|
||||
;;
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
grep_args="${grep_args} $1 $2"
|
||||
shift 2
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
endofopts=1
|
||||
;;
|
||||
-)
|
||||
hyphen=1
|
||||
shift
|
||||
;;
|
||||
-h)
|
||||
silent=1
|
||||
shift
|
||||
;;
|
||||
-*)
|
||||
grep_args="${grep_args} $1"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
# pattern to grep for
|
||||
endofopts=1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# if no -e option was found, take next argument as grep-pattern
|
||||
if [ ${pattern_found} -lt 1 ]
|
||||
then
|
||||
if [ $# -ge 1 ]; then
|
||||
pattern="$1"
|
||||
shift
|
||||
elif [ ${hyphen} -gt 0 ]; then
|
||||
pattern="-"
|
||||
else
|
||||
echo "${prg}: missing pattern" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# call grep ...
|
||||
if [ $# -lt 1 ]
|
||||
then
|
||||
# ... on stdin
|
||||
${zcat} -fq - | ${grep} ${grep_args} -- "${pattern}" -
|
||||
else
|
||||
# ... on all files given on the command line
|
||||
if [ ${silent} -lt 1 -a $# -gt 1 ]; then
|
||||
grep_args="-H ${grep_args}"
|
||||
fi
|
||||
while [ $# -gt 0 ]
|
||||
do
|
||||
${zcat} -fq -- "$1" | ${grep} --label="${1}" ${grep_args} -- "${pattern}" -
|
||||
shift
|
||||
done
|
||||
fi
|
||||
|
||||
exit 0
|
||||
1
programs/zstdless
Executable file
1
programs/zstdless
Executable file
@ -0,0 +1 @@
|
||||
zstdcat $@ | less
|
||||
@ -100,8 +100,10 @@ ls tmp.zst && die "tmp.zst should not be created"
|
||||
|
||||
|
||||
$ECHO "\n**** Pass-Through mode **** "
|
||||
$ECHO "Hello world !" | $ZSTD -df
|
||||
$ECHO "Hello world !" | $ZSTD -dcf
|
||||
$ECHO "Hello world 1!" | $ZSTD -df
|
||||
$ECHO "Hello world 2!" | $ZSTD -dcf
|
||||
$ECHO "Hello world 3!" > tmp1
|
||||
$ZSTD -dcf tmp1
|
||||
|
||||
|
||||
$ECHO "\n**** frame concatenation **** "
|
||||
|
||||
@ -130,7 +130,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
U32 testNb=0;
|
||||
ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
|
||||
ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
|
||||
ZSTD_inBuffer inBuff;
|
||||
ZSTD_inBuffer inBuff, inBuff2;
|
||||
ZSTD_outBuffer outBuff;
|
||||
|
||||
/* Create compressible test buffer */
|
||||
@ -183,12 +183,22 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Basic decompression test */
|
||||
inBuff2 = inBuff;
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
|
||||
{ size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000000000); /* large limit */
|
||||
if (ZSTD_isError(r)) goto _output_error; }
|
||||
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
{ size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Re-use without init */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress again without init (re-use previous settings): ", testNb++);
|
||||
outBuff.pos = 0;
|
||||
{ size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2);
|
||||
if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
@ -553,8 +563,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
/* multi - fragments decompression test */
|
||||
if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
|
||||
CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
|
||||
} else
|
||||
} else {
|
||||
ZSTD_initDStream_usingDict(zd, dict, dictSize);
|
||||
}
|
||||
{ size_t decompressionResult = 1;
|
||||
ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
|
||||
ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user