Docker backend should retry to delete volume on "in use" error (#6381)

pipeline/backend/docker/convert.go

@@ -232,3 +232,7 @@ func splitVolumeParts(volumeParts string) ([]string, error) {
 	}
 	return strings.Split(volumeParts, ":"), nil
 }
+
+func toRef[T any](v T) *T {
+	return &v
+}

pipeline/backend/docker/docker.go

@@ -22,8 +22,9 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"strings"
+	"time"
 
+	"github.com/cenkalti/backoff/v5"
 	"github.com/containerd/errdefs"
 	"github.com/docker/go-connections/tlsconfig"
 	"github.com/moby/moby/api/pkg/stdcopy"
@@ -41,7 +42,11 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v3/shared/utils"
 )
 
-var containerKillTimeout = 5 // seconds
+const (
+	containerKillTimeout               = 5 // seconds
+	volumeRetryWait      time.Duration = 1 * time.Second
+	maxRetry             uint          = 3
+)
 
 type docker struct {
 	client client.APIClient
@@ -339,7 +344,7 @@ func (e *docker) DestroyStep(ctx context.Context, step *backend_types.Step, task
 
 	// we first signal to the container to stop ...
 	if _, err := e.client.ContainerStop(ctx, containerName, client.ContainerStopOptions{
-		Timeout: &containerKillTimeout,
+		Timeout: toRef(containerKillTimeout),
 	}); err != nil && !isErrContainerNotFoundOrNotRunning(err) {
 		// we do not return error yet as we try to kill it first
 		stopErr = fmt.Errorf("could not stop container '%s': %w", step.Name, err)
@@ -377,11 +382,24 @@ func (e *docker) DestroyWorkflow(ctx context.Context, conf *backend_types.Config
 		log.Error().Err(err).Msgf("could not destroy all containers")
 	}
 
-	if _, err := e.client.VolumeRemove(ctx, conf.Volume, client.VolumeRemoveOptions{
-		Force: true,
-	}); err != nil {
+	var err error
+	_, _ = backoff.Retry(ctx, func() (any, error) {
+		_, err = e.client.VolumeRemove(ctx, conf.Volume, client.VolumeRemoveOptions{
+			Force: true,
+		})
+		if err == nil || !isErrVolumeInUse(err) {
+			// if it worked or if we have no "in use error" do not retry
+			return nil, nil
+		}
+		return nil, err
+	}, backoff.WithMaxTries(maxRetry), backoff.WithBackOff(&backoff.ExponentialBackOff{
+		InitialInterval: volumeRetryWait,
+		Multiplier:      2, //nolint:mnd
+	}))
+	if err != nil {
 		log.Error().Err(err).Msgf("could not remove volume '%s'", conf.Volume)
 	}
+
 	if _, err := e.client.NetworkRemove(ctx, conf.Network, client.NetworkRemoveOptions{}); err != nil {
 		log.Error().Err(err).Msgf("could not remove network '%s'", conf.Network)
 	}
@@ -394,19 +412,6 @@ var removeOpts = client.ContainerRemoveOptions{
 	Force:         false,
 }
 
-func isErrContainerNotFoundOrNotRunning(err error) bool {
-	// Error response from daemon: Cannot kill container: ...: No such container: ...
-	// Error response from daemon: Cannot kill container: ...: Container ... is not running"
-	// Error response from podman daemon: can only kill running containers. ... is in state exited
-	// Error response from daemon: removal of container ... is already in progress
-	// Error: No such container: ...
-	return err != nil &&
-		(strings.Contains(err.Error(), "No such container") ||
-			strings.Contains(err.Error(), "is not running") ||
-			strings.Contains(err.Error(), "can only kill running containers") ||
-			(strings.Contains(err.Error(), "removal of container") && strings.Contains(err.Error(), "is already in progress")))
-}
-
 // normalizeArchType converts the arch type reported by docker info into
 // the runtime.GOARCH format
 // TODO: find out if we we need to convert other arch types too

pipeline/backend/docker/errors.go

@@ -0,0 +1,35 @@
+// Copyright 2026 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package docker
+
+import "strings"
+
+func isErrContainerNotFoundOrNotRunning(err error) bool {
+	// Error response from daemon: Cannot kill container: ...: No such container: ...
+	// Error response from daemon: Cannot kill container: ...: Container ... is not running"
+	// Error response from podman daemon: can only kill running containers. ... is in state exited
+	// Error response from daemon: removal of container ... is already in progress
+	// Error: No such container: ...
+	return err != nil &&
+		(strings.Contains(err.Error(), "No such container") ||
+			strings.Contains(err.Error(), "is not running") ||
+			strings.Contains(err.Error(), "can only kill running containers") ||
+			(strings.Contains(err.Error(), "removal of container") && strings.Contains(err.Error(), "is already in progress")))
+}
+
+func isErrVolumeInUse(err error) bool {
+	return err != nil &&
+		strings.Contains(err.Error(), "volume is in use")
+}