[CI] Hook cli-tests up to CI

Add cli-tests to `make test`. This adds a `python3` dependency to `make test`, but not `make check`. We could make this dependency optional by skipping the tests if `python3` is not present.
2025-03-06 16:56:49 +02:00 · 2022-01-24 13:52:08 -08:00 · 2022-01-24 13:52:08 -08:00 · 1fc42de86a
commit 1fc42de86a
parent f3096ff6d1
18 changed files with 102 additions and 96 deletions
--- a/tests/Makefile
+++ b/tests/Makefile
@ -297,7 +297,7 @@ check: shortest
 fuzztest: test-fuzzer test-zstream test-decodecorpus

 .PHONY: test
-test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
+test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus test-cli-tests
 ifeq ($(QEMU_SYS),)
 test: test-pool
 endif
@ -322,6 +322,12 @@ test-zstd test-zstd32 test-zstd-nolegacy: datagen
 	file $(ZSTD)
 	EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)

+test-cli-tests: ZSTD = $(PRGDIR)/zstd
+test-cli-tests: zstd datagen
+	file $(ZSTD)
+	./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen
+
+
 test-fullbench: fullbench datagen
 	$(QEMU_SYS) ./fullbench -i1
 	$(QEMU_SYS) ./fullbench -i1 -P0
--- a/tests/cli-tests/basic/help.sh
+++ b/tests/cli-tests/basic/help.sh
@ -1,4 +1,7 @@
-#!/bin/sh -e
+#!/bin/sh
+
+set -e
+
 println "+ zstd -h"
 zstd -h
 println "+ zstd -H"
--- a/tests/cli-tests/basic/version.sh
+++ b/tests/cli-tests/basic/version.sh
@ -1,3 +1,6 @@
-#!/bin/sh -e
+#!/bin/sh
+
+set -e
+
 zstd -V
 zstd --version
--- a/tests/cli-tests/bin/cmp_size
+++ b/tests/cli-tests/bin/cmp_size
@ -1,7 +1,5 @@
 #!/bin/sh

-# Small utility to
-
 set -e

 usage()
--- a/tests/cli-tests/bin/println
+++ b/tests/cli-tests/bin/println
@ -1,2 +1,2 @@
-#!/bin/env sh
+#!/bin/sh
 printf '%b\n' "${*}"
--- a/tests/cli-tests/common/format.sh
+++ b/tests/cli-tests/common/format.sh
@ -1,6 +1,6 @@
 #!/bin/sh

-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"

 zstd_supports_format()
 {
--- a/tests/cli-tests/common/mtime.sh
+++ b/tests/cli-tests/common/mtime.sh
@ -1,4 +1,4 @@
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"

 MTIME="stat -c %Y"
 case "$UNAME" in
--- a/tests/cli-tests/common/permissions.sh
+++ b/tests/cli-tests/common/permissions.sh
@ -1,4 +1,4 @@
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"

 GET_PERMS="stat -c %a"
 case "$UNAME" in
--- a/tests/cli-tests/compression/basic.sh
+++ b/tests/cli-tests/compression/basic.sh
@ -1,7 +1,9 @@
-#!/bin/sh -e
+#!/bin/sh

-# Uncomment the set -x line for debugging
-# set -x
+set -e
+
+# Uncomment the set -v line for debugging
+# set -v

 # Test compression flags and check that they work
 zstd file                 ; zstd -t file.zst
--- a/tests/cli-tests/compression/format.sh
+++ b/tests/cli-tests/compression/format.sh
@ -1,6 +1,6 @@
 #!/bin/sh

-source "$COMMON/format.sh"
+. "$COMMON/format.sh"

 set -e

--- a/tests/cli-tests/compression/levels.sh
+++ b/tests/cli-tests/compression/levels.sh
@ -1,7 +1,7 @@
 #!/bin/sh

 set -e
-set -x
+set -v

 datagen > file

@ -12,7 +12,7 @@ zstd -1 file -o file-1.zst
 zstd -19 file -o file-19.zst
 zstd -22 --ultra file -o file-22.zst

-zstd -t file-{f10,f1,1,19,22}.zst
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst

 cmp_size -ne file-19.zst file-22.zst
 cmp_size -lt file-19.zst file-1.zst
@ -41,10 +41,10 @@ ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
 ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
 ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst

-cmp file-f10{,-env}.zst || die "Environment variable failed to set level"
-cmp file-1{,-env}.zst   || die "Environment variable failed to set level"
-cmp file-19{,-env}.zst  || die "Environment variable failed to set level"
-cmp file-99{,-env}.zst  || die "Environment variable failed to set level"
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"

 # Test invalid environment clevel is the default level
 zstd -f file
@ -60,5 +60,5 @@ ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
 ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
 ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst

-cmp file-1{,-env}.zst  || die "Environment variable not overridden"
-cmp file-f1{,-env}.zst || die "Environment variable not overridden"
+cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
--- a/tests/cli-tests/compression/levels.sh.stderr.exact
+++ b/tests/cli-tests/compression/levels.sh.stderr.exact
@ -1,75 +1,71 @@
-+ datagen
-+ zstd --fast=10 file -o file-f10.zst
-+ zstd --fast=1 file -o file-f1.zst
-+ zstd -1 file -o file-1.zst
-+ zstd -19 file -o file-19.zst
-+ zstd -22 --ultra file -o file-22.zst
-+ zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
-+ cmp_size -ne file-19.zst file-22.zst
-+ cmp_size -lt file-19.zst file-1.zst
-+ cmp_size -lt file-1.zst file-f1.zst
-+ cmp_size -lt file-f1.zst file-f10.zst
-+ zstd --fast file -f
-+ cmp file.zst file-f1.zst
-+ zstd -0 file -o file-0.zst
-+ zstd -f file
-+ cmp file.zst file-0.zst
-+ zstd -99 file -o file-99.zst
+
+datagen > file
+
+# Compress with various levels and ensure that their sizes are ordered
+zstd --fast=10 file -o file-f10.zst
+zstd --fast=1 file -o file-f1.zst
+zstd -1 file -o file-1.zst
+zstd -19 file -o file-19.zst
+zstd -22 --ultra file -o file-22.zst
+
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
+
+cmp_size -ne file-19.zst file-22.zst
+cmp_size -lt file-19.zst file-1.zst
+cmp_size -lt file-1.zst file-f1.zst
+cmp_size -lt file-f1.zst file-f10.zst
+
+# Test default levels
+zstd --fast file -f
+cmp file.zst file-f1.zst || die "--fast is not level -1"
+
+zstd -0 file -o file-0.zst
+zstd -f file
+cmp file.zst file-0.zst || die "Level 0 is not the default level"
+
+# Test level clamping
+zstd -99 file -o file-99.zst
 Warning : compression level higher than max, reduced to 19 
-+ cmp file-19.zst file-99.zst
-+ zstd --fast=200000 file -c
-+ zstd -t
-+ zstd -5000000000 -f file
+cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
+zstd --fast=200000 file -c | zstd -t
+
+zstd -5000000000 -f file       && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
-+ :
-+ zstd --fast=5000000000 -f file
+zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
-+ :
-+ ZSTD_CLEVEL=-10
-+ zstd file -o file-f10-env.zst
-+ ZSTD_CLEVEL=1
-+ zstd file -o file-1-env.zst
-+ ZSTD_CLEVEL=+19
-+ zstd file -o file-19-env.zst
-+ ZSTD_CLEVEL=+99
-+ zstd file -o file-99-env.zst
+
+# Test setting a level through the environment variable
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst
 Warning : compression level higher than max, reduced to 19 
-+ cmp file-f10.zst file-f10-env.zst
-+ cmp file-1.zst file-1-env.zst
-+ cmp file-19.zst file-19-env.zst
-+ cmp file-99.zst file-99-env.zst
-+ zstd -f file
-+ ZSTD_CLEVEL=-
-+ zstd -f file -o file-env.zst
+
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
+
+# Test invalid environment clevel is the default level
+zstd -f file
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=+
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=-a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=+a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=3a7
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst       ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=3a7: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=5000000000
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=5000000000: numeric value too large 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=10
-+ zstd -f file -1 -o file-1-env.zst
-+ ZSTD_CLEVEL=10
-+ zstd -f file --fast=1 -o file-f1-env.zst
-+ cmp file-1.zst file-1-env.zst
-+ cmp file-f1.zst file-f1-env.zst
+
+# Test environment clevel is overridden by command line
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst
+
+cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
--- a/tests/cli-tests/dict-builder/no-inputs.sh
+++ b/tests/cli-tests/dict-builder/no-inputs.sh
@ -1,3 +1,3 @@
 #!/bin/sh
-set -x
+set -v
 zstd --train
--- a/tests/cli-tests/dict-builder/no-inputs.sh.exit
+++ b/tests/cli-tests/dict-builder/no-inputs.sh.exit
--- a/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
+++ b/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
@ -1,4 +1,4 @@
-+ zstd --train
+zstd --train
 !  Warning : nb of samples too low for proper processing ! 
 !  Please provide _one file per sample_. 
 !  Alternatively, split files into fixed-size blocks representative of samples, with -B# 
--- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh
+++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh
@ -1,6 +1,6 @@
 #!/bin/sh

-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"

 set -e

@ -22,7 +22,7 @@ if [ false ]; then
 	datagen -g1000 -s0 > file0
 fi

-set -x
+set -v
 zstd files/0 -D dicts/0
 zstd -t files/0.zst -D dicts/0
 zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
--- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
+++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
@ -1,8 +1,6 @@
-+ zstd files/0 -D dicts/0
-+ zstd -t files/0.zst -D dicts/0
-+ zstd -t files/0.zst -D dicts/1
+zstd files/0 -D dicts/0
+zstd -t files/0.zst -D dicts/0
+zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
 files/0.zst : Decoding error (36) : Dictionary mismatch 
-+ :
-+ zstd -t files/0.zst
+zstd -t files/0.zst            && die "Must fail" ||:
 files/0.zst : Decoding error (36) : Dictionary mismatch 
-+ :
--- a/tests/cli-tests/dictionaries/setup_once
+++ b/tests/cli-tests/dictionaries/setup_once
@ -2,7 +2,7 @@

 set -e

-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"


 mkdir files/ dicts/