From 06bb25c980569445e860916279e3a4e75ce0d879 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Fri, 6 Jun 2025 17:46:25 +0200 Subject: [PATCH] Defining test sandbox in compose file allows testing against multi node swarm setup (#591) * Defining test sandbox in compose file allows swapping with multi node swarm setup * Test cases can request a multi node swarm cluster * Docker healthchecks can be used for awaiting ready state * Compose profiles can be used --- test/README.md | 15 ++++++------ test/collision/.swarm | 0 test/collision/run.sh | 2 -- test/docker-compose.yml | 27 +++++++++++++++++++++ test/secrets/.swarm | 0 test/secrets/run.sh | 2 -- test/services/.swarm | 0 test/services/run.sh | 2 -- test/swarm/.swarm | 0 test/swarm/run.sh | 2 -- test/test.sh | 54 +++++++++++++++-------------------------- 11 files changed, 53 insertions(+), 51 deletions(-) create mode 100644 test/collision/.swarm create mode 100644 test/docker-compose.yml create mode 100644 test/secrets/.swarm create mode 100644 test/services/.swarm create mode 100644 test/swarm/.swarm diff --git a/test/README.md b/test/README.md index ec82644..f5b963c 100644 --- a/test/README.md +++ b/test/README.md @@ -39,14 +39,6 @@ Setting this value lets you run tests against different existing images, so you IMAGE_TAG=v2.30.0 ./test.sh ``` -#### `NO_IMAGE_CACHE` - -When set, images from remote registries will not be cached and shared between sandbox containers. - -```sh -NO_IMAGE_CACHE=1 ./test.sh -``` - By default, two local images are created that persist the image data and provide it to containers at runtime. ## Understanding the test setup @@ -68,3 +60,10 @@ cd "$(dirname "$0")" . ../util.sh current_test=$(basename $(pwd)) ``` + +### Running tests in swarm mode + +A test case can signal it wants to run in swarm mode by placing an empty `.swarm` file inside the directory. +In case the swarm setup should be compose of multiple nodes, a `.multinode` file can be used. + +A multinode setup will contain one manager (`manager`) and two worker nodes (`worker1` and `worker2`). diff --git a/test/collision/.swarm b/test/collision/.swarm new file mode 100644 index 0000000..e69de29 diff --git a/test/collision/run.sh b/test/collision/run.sh index 8948feb..a6d2899 100755 --- a/test/collision/run.sh +++ b/test/collision/run.sh @@ -8,8 +8,6 @@ current_test=$(basename $(pwd)) export LOCAL_DIR=$(mktemp -d) -docker swarm init - docker stack deploy --compose-file=docker-compose.yml test_stack while [ -z $(docker ps -q -f name=backup) ]; do diff --git a/test/docker-compose.yml b/test/docker-compose.yml new file mode 100644 index 0000000..ec7bd69 --- /dev/null +++ b/test/docker-compose.yml @@ -0,0 +1,27 @@ +services: + manager: &node + privileged: true + image: offen/docker-volume-backup:test-sandbox + healthcheck: + test: ["CMD", "docker", "info"] + interval: 1s + timeout: 5s + retries: 50 + volumes: + - $SOURCE:/code + - $TARBALL:/cache/image.tar.gz + - docker_volume_backup_test_sandbox_image:/var/lib/docker/image + - docker_volume_backup_test_sandbox_overlay2:/var/lib/docker/overlay2 + + worker1: + <<: *node + profiles: + - multinode + worker2: + <<: *node + profiles: + - multinode + +volumes: + docker_volume_backup_test_sandbox_image: + docker_volume_backup_test_sandbox_overlay2: diff --git a/test/secrets/.swarm b/test/secrets/.swarm new file mode 100644 index 0000000..e69de29 diff --git a/test/secrets/run.sh b/test/secrets/run.sh index dc0569c..f972f5c 100755 --- a/test/secrets/run.sh +++ b/test/secrets/run.sh @@ -6,8 +6,6 @@ cd $(dirname $0) . ../util.sh current_test=$(basename $(pwd)) -docker swarm init - printf "test" | docker secret create minio_root_user - printf "GMusLtUmILge2by+z890kQ" | docker secret create minio_root_password - diff --git a/test/services/.swarm b/test/services/.swarm new file mode 100644 index 0000000..e69de29 diff --git a/test/services/run.sh b/test/services/run.sh index ad1f8c0..b9e9f46 100755 --- a/test/services/run.sh +++ b/test/services/run.sh @@ -6,8 +6,6 @@ cd $(dirname $0) . ../util.sh current_test=$(basename $(pwd)) -docker swarm init - docker stack deploy --compose-file=docker-compose.yml test_stack while [ -z $(docker ps -q -f name=backup) ]; do diff --git a/test/swarm/.swarm b/test/swarm/.swarm new file mode 100644 index 0000000..e69de29 diff --git a/test/swarm/run.sh b/test/swarm/run.sh index ad1f8c0..b9e9f46 100755 --- a/test/swarm/run.sh +++ b/test/swarm/run.sh @@ -6,8 +6,6 @@ cd $(dirname $0) . ../util.sh current_test=$(basename $(pwd)) -docker swarm init - docker stack deploy --compose-file=docker-compose.yml test_stack while [ -z $(docker ps -q -f name=backup) ]; do diff --git a/test/test.sh b/test/test.sh index 390e7e3..8d4c0df 100755 --- a/test/test.sh +++ b/test/test.sh @@ -7,17 +7,13 @@ IMAGE_TAG=${IMAGE_TAG:-canary} sandbox="docker_volume_backup_test_sandbox" tarball="$(mktemp -d)/image.tar.gz" +compose_profile="default" trap finish EXIT INT TERM finish () { rm -rf $(dirname $tarball) - if [ ! -z $(docker ps -aq --filter=name=$sandbox) ]; then - docker rm -f $(docker stop $sandbox) - fi - if [ ! -z $(docker volume ls -q --filter=name="^${sandbox}\$") ]; then - docker volume rm $sandbox - fi + docker compose --profile $compose_profile down } docker build -t offen/docker-volume-backup:test-sandbox . @@ -41,41 +37,29 @@ for dir in $(find $find_args | sort); do echo "" test="${dir}/run.sh" - docker_run_args="--name "$sandbox" --detach \ - --privileged \ - -v $(dirname $(pwd)):/code \ - -v $tarball:/cache/image.tar.gz \ - -v $sandbox:/var/lib/docker" + export TARBALL=$tarball + export SOURCE=$(dirname $(pwd)) - if [ -z "$NO_IMAGE_CACHE" ]; then - docker_run_args="$docker_run_args \ - -v "${sandbox}_image":/var/lib/docker/image \ - -v "${sandbox}_overlay2":/var/lib/docker/overlay2" + if [ -f ${dir}/.multinode ]; then + compose_profile="multinode" fi - docker run $docker_run_args offen/docker-volume-backup:test-sandbox + docker compose --profile $compose_profile up -d --wait - retry_counter=0 - until timeout 5 docker exec $sandbox /bin/sh -c 'docker info' > /dev/null 2>&1; do - if [ $retry_counter -gt 20 ]; then - echo "Gave up waiting for Docker daemon to become ready after 20 attempts" - exit 1 - fi + if [ -f "${dir}/.swarm" ]; then + docker compose exec manager docker swarm init + elif [ -f "${dir}/.multinode" ]; then + docker compose exec manager docker swarm init + manager_ip=$(docker compose exec manager docker node inspect $(docker compose exec manager docker node ls -q) --format '{{ .Status.Addr }}') + token=$(docker compose exec manager docker swarm join-token -q worker) + docker compose exec worker1 docker swarm join --token $token $manager_ip:2377 + docker compose exec worker2 docker swarm join --token $token $manager_ip:2377 + fi - if [ "$(docker inspect $sandbox --format '{{ .State.Running }}')" = "false" ]; then - docker rm $sandbox - docker run $docker_run_args offen/docker-volume-backup:test-sandbox - fi + docker compose exec manager /bin/sh -c "docker load -i /cache/image.tar.gz" + docker compose exec -e TEST_VERSION=$IMAGE_TAG manager /bin/sh -c "/code/test/$test" - sleep 0.5 - retry_counter=$((retry_counter+1)) - done - - docker exec $sandbox /bin/sh -c "docker load -i /cache/image.tar.gz" - docker exec -e TEST_VERSION=$IMAGE_TAG $sandbox /bin/sh -c "/code/test/$test" - - docker rm $(docker stop $sandbox) - docker volume rm $sandbox + docker compose --profile $compose_profile down echo "" echo "$test passed" echo ""