Current test setup is too restrictive, allow running a series of executables in different contexts instead (#597)

* Current test setup is too restrictive, allow running a series of executables in different contexts instead

* Split existing tests that test against swarm

test/README.md

@@ -49,8 +49,8 @@ As the sandbox container is also expected to be torn down post test, the scripts
 
 ## Anatomy of a test case
 
-The `test.sh` script looks for an exectuable file called `run.sh` in each directory.
-When found, it is executed and signals success by returning a 0 exit code.
+The `test.sh` script looks for all exectuable files in each directory.
+When found, all of them are executed in series and are expected to signal success by returning a 0 exit code.
 Any other exit code is considered a failure and will halt execution of further tests.
 
 There is an `util.sh` file containing a few commonly used helpers which can be used by putting the following prelude to a new test case:
@@ -68,10 +68,5 @@ In case the swarm setup should be compose of multiple nodes, a `.multinode` file
 
 A multinode setup will contain one manager (`manager`) and two worker nodes (`worker1` and `worker2`).
 
-If a test is expected to run in the context of a node other than the `manager`, the hostname can be put in the `.multinode` file.
-
-> [!IMPORTANT]
-> When running against a multi-node setup and targeting a non-manager node, the test script will automatically deploy a stack named `test_stack` based on the compose file in the test directory.
-> This is required because the non-manager node cannot deploy the stack itself from within the test script.
-> This also means, you cannot mount local directories created in your test script, as the containers are already created when the script runs.
-> You can work around this limitation by creating named volumes and then `docker cp`ing the contents your test needs to inspect.
+If a test is expected to run in the context of a node other than the `manager`, you can create a `.context` file containing the name of the node you want the test to run in.
+E.g. if your script `02run.sh` is expected to be run on `worker2`, create a file called `02run.sh.context` with the content `worker2`

test/commands/01compose.sh

@@ -31,32 +31,3 @@ fi
 pass "Did not find unexpected file."
 
 docker compose down --volumes
-
-info "Running commands test in swarm mode next."
-
-export LOCAL_DIR=$(mktemp -d)
-export TMP_DIR=$(mktemp -d)
-
-docker swarm init
-
-docker stack deploy --compose-file=docker-compose.yml test_stack
-
-while [ -z $(docker ps -q -f name=backup) ]; do
-  info "Backup container not ready yet. Retrying."
-  sleep 1
-done
-
-sleep 20
-
-docker exec $(docker ps -q -f name=backup) backup
-
-tar -xvf "$LOCAL_DIR/test.tar.gz" -C $TMP_DIR
-if [ ! -f "$TMP_DIR/backup/data/dump.sql" ]; then
-  fail "Could not find file written by pre command."
-fi
-pass "Found expected file."
-
-if [ -f "$TMP_DIR/backup/data/post.txt" ]; then
-  fail "File created in post command was present in backup."
-fi
-pass "Did not find unexpected file."

test/commands/02swarm.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+export TMP_DIR=$(mktemp -d)
+
+docker swarm init
+
+docker stack deploy --compose-file=docker-compose.yml test_stack
+
+while [ -z $(docker ps -q -f name=backup) ]; do
+  info "Backup container not ready yet. Retrying."
+  sleep 1
+done
+
+sleep 20
+
+docker exec $(docker ps -q -f name=backup) backup
+
+tar -xvf "$LOCAL_DIR/test.tar.gz" -C $TMP_DIR
+if [ ! -f "$TMP_DIR/backup/data/dump.sql" ]; then
+  fail "Could not find file written by pre command."
+fi
+pass "Found expected file."
+
+if [ -f "$TMP_DIR/backup/data/post.txt" ]; then
+  fail "File created in post command was present in backup."
+fi
+pass "Did not find unexpected file."

test/proxy/01compose.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+export LOCAL_DIR=$(mktemp -d)
+
+docker compose up -d --quiet-pull
+sleep 5
+
+# The default configuration in docker-compose.yml should
+# successfully create a backup.
+docker compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then
+  fail "Archive was not created"
+fi
+pass "Found relevant archive file."
+
+# Disabling POST should make the backup run fail
+ALLOW_POST="0" docker compose up -d
+sleep 5
+
+set +e
+docker compose exec backup backup
+if [ $? = "0" ]; then
+  fail "Expected invocation to exit non-zero."
+fi
+set -e
+pass "Invocation exited non-zero."
+
+docker compose down --volumes

test/proxy/02swarm.sh

@@ -6,40 +6,6 @@ cd $(dirname $0)
 . ../util.sh
 current_test=$(basename $(pwd))
 
-export LOCAL_DIR=$(mktemp -d)
-
-docker compose up -d --quiet-pull
-sleep 5
-
-# The default configuration in docker-compose.yml should
-# successfully create a backup.
-docker compose exec backup backup
-
-sleep 5
-
-expect_running_containers "3"
-
-if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then
-  fail "Archive was not created"
-fi
-pass "Found relevant archive file."
-
-# Disabling POST should make the backup run fail
-ALLOW_POST="0" docker compose up -d
-sleep 5
-
-set +e
-docker compose exec backup backup
-if [ $? = "0" ]; then
-  fail "Expected invocation to exit non-zero."
-fi
-set -e
-pass "Invocation exited non-zero."
-
-docker compose down --volumes
-
-# Next, the test is run against a Swarm setup
-
 docker swarm init
 
 export LOCAL_DIR=$(mktemp -d)

test/test.sh

@@ -36,7 +36,6 @@ for dir in $(find $find_args | sort); do
   echo "################################################"
   echo ""
 
-  test="${dir}/run.sh"
   export TARBALL=$tarball
   export SOURCE=$(dirname $(pwd))
 
@@ -45,13 +44,6 @@ for dir in $(find $find_args | sort); do
   fi
 
   docker compose --profile $compose_profile up -d --wait
-  test_context=manager
-  if [ -f "${dir}/.multinode" ] && [ -s "${dir}/.multinode" ]; then
-    test_context=$(cat $dir/.multinode)
-    echo "Running tests on $test_context instead of manager"
-  fi
-  docker compose exec $test_context /bin/sh -c "docker load -i /cache/image.tar.gz"
-
   if [ -f "${dir}/.swarm" ]; then
     docker compose exec manager docker swarm init
   elif [ -f "${dir}/.multinode" ]; then
@@ -60,16 +52,22 @@ for dir in $(find $find_args | sort); do
     token=$(docker compose exec manager docker swarm join-token -q worker)
     docker compose exec worker1 docker swarm join --token $token $manager_ip:2377
     docker compose exec worker2 docker swarm join --token $token $manager_ip:2377
-
-    if [ "$test_context" != "manager" ]; then
-      docker compose exec -w "/code/$dir" manager docker stack deploy --compose-file="docker-compose.yml" test_stack
-    fi
   fi
 
-  docker compose exec -e TEST_VERSION=$IMAGE_TAG $test_context /bin/sh -c "/code/$test"
+  for svc in $(docker compose ps -q); do
+    docker exec $svc /bin/sh -c "docker load -i /cache/image.tar.gz"
+  done
+
+  for executable in $(find $dir -type f -executable | sort); do
+    context="manager"
+    if [ -f "$executable.context" ]; then
+        context=$(cat "$executable.context")
+    fi
+    docker compose exec -e TEST_VERSION=$IMAGE_TAG $context /bin/sh -c "/code/$executable"
+  done
 
   docker compose --profile $compose_profile down
   echo ""
-  echo "$test passed"
+  echo "$dir passed"
   echo ""
 done

test/worker-node/.multinode

@@ -1 +0,0 @@
-worker1

test/worker-node/01manager.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker stack deploy --compose-file=docker-compose.yml test_stack

test/worker-node/02worker1.sh.context

@@ -0,0 +1 @@
+worker1