krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-07 01:10:04 +02:00
Code Issues Releases 1 Activity

Add explicit --pass-through flag and default to enabled for *cat (#3223)

Browse Source 
Fixes #3211.

Adds the `--[no-]pass-through` flag which enables/disables pass-through mode.

* `zstdcat`, `zcat`, and `gzcat` default to `--pass-through`.
  Pass-through mode can be disabled by passing `--no-pass-through`.
* All other binaries default to not setting pass-through mode.
  However, we preserve the legacy behavior of enabling pass-through
  mode when writing to stdout with `-f` set, unless pass-through
  mode is explicitly disabled with `--no-pass-through`.

Adds a new test for this behavior that should codify the behavior we want.
This commit is contained in:
Nick Terrell 2022-08-04 17:15:59 -07:00 committed by GitHub
parent d0dcc9d775
commit 03cc84fddb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 125 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
programs/fileio.c
View File

@ -290,6 +290,7 @@ FIO_prefs_t* FIO_createPreferences(void)
ret->excludeCompressedFiles = 0;
ret->allowBlockDevices = 0;
ret->asyncIO = AIO_supported();
ret->passThrough = -1;
return ret;
}
@ -463,6 +464,10 @@ void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) {
#endif
}
void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
prefs->passThrough = (value != 0);
}
/* FIO_ctx_t functions */
void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) {
@ -2336,6 +2341,16 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
{
unsigned readSomething = 0;
unsigned long long filesize = 0;
int passThrough = prefs->passThrough;
if (passThrough == -1) {
/* If pass-through mode is not explicitly enabled or disabled,
* default to the legacy behavior of enabling it if we are writing
* to stdout with the overwrite flag enabled.
*/
passThrough = prefs->overwrite && !strcmp(dstFileName, stdoutmark);
}
assert(passThrough == 0 || passThrough == 1);
/* for each frame */
for ( ; ; ) {
@ -2353,7 +2368,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
}
readSomething = 1; /* there is at least 1 byte in srcFile */
if (ress.readCtx->srcBufferLoaded < toRead) { /* not enough input to check magic number */
if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
if (passThrough) {
return FIO_passThrough(&ress);
}
DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName);
@ -2391,7 +2406,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName);
return 1;
#endif
} else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
} else if (passThrough) {
return FIO_passThrough(&ress);
} else {
DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);

1
programs/fileio.h
View File

@ -105,6 +105,7 @@ void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value);
/* FIO_ctx_t functions */
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);

1
programs/fileio_types.h
View File

@ -68,6 +68,7 @@ typedef struct FIO_prefs_s {
int patchFromMode;
int contentSize;
int allowBlockDevices;
int passThrough;
} FIO_prefs_t;
#endif /* FILEIO_TYPES_HEADER */

17
programs/zstdcli.c
View File

@ -264,6 +264,15 @@ static void usage_advanced(const char* programName)
# else
DISPLAYOUT(" --[no-]sparse sparse mode (default: disabled)\n");
# endif
{
char const* passThroughDefault = "disabled";
if (exeNameMatch(programName, ZSTD_CAT) ||
exeNameMatch(programName, ZSTD_ZCAT) ||
exeNameMatch(programName, ZSTD_GZCAT)) {
passThroughDefault = "enabled";
}
DISPLAYOUT(" --[no-]pass-through : passes through uncompressed files as-is (default: %s\n)", passThroughDefault);
}
#endif /* ZSTD_NODECOMPRESS */
#ifndef ZSTD_NODICT
@ -870,14 +879,14 @@ int main(int argCount, const char* argv[])
/* preset behaviors */
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1);
dictCLevel = cLevel = 6; /* gzip default is -6 */
}
if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); } /* behave like gunzip, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like lzma */
if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like xz */
@ -926,6 +935,8 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; }
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; }
if (!strcmp(argument, "--pass-through")) { FIO_setPassThroughFlag(prefs, 1); continue; }
if (!strcmp(argument, "--no-pass-through")) { FIO_setPassThroughFlag(prefs, 0); continue; }
if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
if (!strcmp(argument, "--asyncio")) { FIO_setAsyncIOFlag(prefs, 1); continue;}
if (!strcmp(argument, "--no-asyncio")) { FIO_setAsyncIOFlag(prefs, 0); continue;}

57
tests/cli-tests/decompression/pass-through.sh Executable file
View File

@ -0,0 +1,57 @@
#!/bin/sh
set -e
. "$COMMON/platform.sh"
echo "" > 1
echo "2" > 2
echo "23" > 3
echo "234" > 4
echo "some data" > file
println "+ passthrough enabled"
zstd file
# Test short files
zstd -dc --pass-through 1 2 3 4
# Test *cat symlinks
zstdcat file
"$ZSTD_SYMLINK_DIR/zcat" file
"$ZSTD_SYMLINK_DIR/gzcat" file
# Test multiple files with mix of compressed & not
zstdcat file file.zst
zstdcat file.zst file
# Test --pass-through
zstd -dc --pass-through file
zstd -d --pass-through file -o pass-through-file
# Test legacy implicit passthrough with -fc
zstd -dcf file
zstd -dcf file file.zst
zstd -df < file
zstd -dcf < file file.zst -
zstd -dcf < file.zst file -
$DIFF file pass-through-file
println "+ passthrough disabled"
# Test *cat
zstdcat --no-pass-through file && die "should fail"
"$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail"
"$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail"
# Test zstd without implicit passthrough
zstd -d file -o no-pass-through-file && die "should fail"
zstd -d < file && die "should fail"
# Test legacy implicit passthrough with -fc
zstd --no-pass-through -dcf file && die "should fail"
zstd --no-pass-through -dcf file file.zst && die "should fail"
zstd --no-pass-through -df < file && die "should fail"
zstd --no-pass-through -dcf < file file.zst - && die "should fail"
zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||:

10
tests/cli-tests/decompression/pass-through.sh.stderr.exact Normal file
View File

@ -0,0 +1,10 @@
zstd: file: unsupported format
zstd: file: unsupported format
zstd: file: unsupported format
zstd: file: unsupported format
zstd: /*stdin*\: unsupported format
zstd: file: unsupported format
zstd: file: unsupported format
zstd: /*stdin*\: unsupported format
zstd: /*stdin*\: unsupported format
zstd: file: unsupported format

25
tests/cli-tests/decompression/pass-through.sh.stdout.exact Normal file
View File

@ -0,0 +1,25 @@
+ passthrough enabled
2
23
234
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
some data
+ passthrough disabled
some data
some data
some data