Merge pull request #860 from felixhandte/zstd-lz4-support-tests

Add Default LZ4 Support When Available

.travis.yml

@@ -24,6 +24,8 @@ matrix:
     - env: Cmd='make -j uasanregressiontest'
     - env: Cmd='make -j msanregressiontest'
 
+    - env: Cmd='make lz4install && make -C tests test-lz4'
+
 git:
   depth: 1
 

Makefile

@@ -98,6 +98,7 @@ clean:
 	@$(MAKE) -C examples/ $@ > $(VOID)
 	@$(MAKE) -C contrib/gen_html $@ > $(VOID)
 	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
+	@$(RM) -r lz4
 	@echo Cleaning completed
 
 #------------------------------------------------------------------------------
@@ -286,6 +287,10 @@ gpp6install: apt-add-repo
 clang38install:
 	APT_PACKAGES="clang-3.8" $(MAKE) apt-install
 
+# Ubuntu 14.04 ships a too-old lz4
+lz4install:
+	[ -e lz4 ] || git clone https://github.com/lz4/lz4 && sudo $(MAKE) -C lz4 install
+
 endif
 
 

programs/Makefile

@@ -140,13 +140,10 @@ allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy
 
 $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
 
-zstd zstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP)
-zstd zstd4 : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD)
-zstd4 : CPPFLAGS += $(LZ4CPP)
-zstd4 : LDFLAGS += $(LZ4LD)
-zstd : LZ4_MSG := - lz4 support is disabled
-zstd zstd4 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd zstd4 : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
+zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
+zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD)
+zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+zstd : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
 	@echo "$(THREAD_MSG)"
 	@echo "$(ZLIB_MSG)"
 	@echo "$(LZMA_MSG)"

programs/README.md

@@ -40,6 +40,16 @@ There are however other Makefile targets that create different variations of CLI
   In which case, linking stage will fail if `lzma` library cannot be found.
   This might be useful to prevent silent feature disabling.
 
+- __HAVE_LZ4__ : `zstd` can compress and decompress files in `.lz4` formats.
+  This is ordered through commands `--format=lz4`.
+  Alternatively, symlinks named `lz4`, or `unlz4` will mimic intended behavior.
+  `.lz4` support is automatically enabled when `lz4` library is detected at build time.
+  It's possible to disable `.lz4` support, by setting HAVE_LZ4=0 .
+  Example : make zstd HAVE_LZ4=0
+  It's also possible to force compilation with lz4 support, using HAVE_LZ4=1.
+  In which case, linking stage will fail if `lz4` library cannot be found.
+  This might be useful to prevent silent feature disabling.
+
 - __ZSTD_LEGACY_SUPPORT__ : `zstd` can decompress files compressed by older versions of `zstd`.
   Starting v0.8.0, all versions of `zstd` produce frames compliant with the [specification](../doc/zstd_compression_format.md), and are therefore compatible.
   But older versions (< v0.8.0) produced different, incompatible, frames.

programs/fileio.c

@@ -57,6 +57,7 @@
 
 #define LZ4_MAGICNUMBER 0x184D2204
 #if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS)
+#  define LZ4F_ENABLE_OBSOLETE_ENUMS
 #  include <lz4frame.h>
 #  include <lz4.h>
 #endif

programs/zstd.1

@@ -60,11 +60,11 @@ In most places where an integer argument is expected, an optional suffix is supp
 .
 .TP
 \fBKiB\fR
-Multiply the integer by 1,024 (2^10)\. \fBKi\fR, \fBK\fR, and \fBKB\fR are accepted as synonyms for \fBKiB\fR\.
+Multiply the integer by 1,024 (2\e \fBKi\fR, \fBK\fR, and \fBKB\fR are accepted as synonyms for \fBKiB\fR\.
 .
 .TP
 \fBMiB\fR
-Multiply the integer by 1,048,576 (2^20)\. \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\.
+Multiply the integer by 1,048,576 (2\e \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\.
 .
 .SS "Operation mode"
 If multiple operation mode options are given, the last one takes effect\.
@@ -151,6 +151,10 @@ keep source file(s) after successful compression or decompression\. This is the
 operate recursively on dictionaries
 .
 .TP
+\fB\-\-format=FORMAT\fR
+compress and decompress in other formats\. If compiled with support, zstd can compress to or decompress from other compression algorithm formats\. Possibly available options are \fBgzip\fR, \fBxz\fR, \fBlzma\fR, and \fBlz4\fR\.
+.
+.TP
 \fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR
 display help/long help and exit
 .

programs/zstd.1.md

@@ -147,6 +147,10 @@ the last one takes effect.
     This is the default behavior.
 * `-r`:
     operate recursively on dictionaries
+* `--format=FORMAT`:
+    compress and decompress in other formats. If compiled with
+    support, zstd can compress to or decompress from other compression algorithm
+    formats. Possibly available options are `gzip`, `xz`, `lzma`, and `lz4`.
 * `-h`/`-H`, `--help`:
     display help/long help and exit
 * `-V`, `--version`:

programs/zstdcli.c

@@ -61,6 +61,8 @@
 #define ZSTD_UNLZMA "unlzma"
 #define ZSTD_XZ "xz"
 #define ZSTD_UNXZ "unxz"
+#define ZSTD_LZ4 "lz4"
+#define ZSTD_UNLZ4 "unlz4"
 
 #define KB *(1 <<10)
 #define MB *(1 <<20)
@@ -426,6 +428,8 @@ int main(int argCount, const char* argv[])
     if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); }    /* behave like unlzma */
     if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); }    /* behave like xz */
     if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); }    /* behave like unxz */
+    if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); }    /* behave like xz */
+    if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); }    /* behave like unxz */
     memset(&compressionParams, 0, sizeof(compressionParams));
 
     /* command switches */

tests/Makefile

@@ -374,4 +374,37 @@ test-decodecorpus-cli: decodecorpus
 test-pool: poolTests
 	$(QEMU_SYS) ./poolTests
 
+test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd
+test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4
+test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4
+test-lz4: zstd decodecorpus
+	ln -s $(PRGDIR)/zstd lz4
+	ln -s $(PRGDIR)/zstd unlz4
+
+	./decodecorpus -ptmp
+	# lz4 -> zstd
+	lz4 < tmp | \
+	$(ZSTD) -d | \
+	cmp - tmp
+	lz4 < tmp | \
+	$(ZSTD_UNLZ4) | \
+	cmp - tmp
+	# zstd -> lz4
+	$(ZSTD) --format=lz4 < tmp | \
+	lz4 -d | \
+	cmp - tmp
+	$(ZSTD_LZ4) < tmp | \
+	lz4 -d | \
+	cmp - tmp
+	# zstd -> zstd
+	$(ZSTD) --format=lz4 < tmp | \
+	$(ZSTD) -d | \
+	cmp - tmp
+	# zstd -> zstd
+	$(ZSTD) < tmp | \
+	$(ZSTD) -d | \
+	cmp - tmp
+
+	rm tmp lz4 unlz4
+
 endif