diff --git a/pipeline/backend/docker/convert.go b/pipeline/backend/docker/convert.go index 25b0790fc3..3acb1a9699 100644 --- a/pipeline/backend/docker/convert.go +++ b/pipeline/backend/docker/convert.go @@ -232,3 +232,7 @@ func splitVolumeParts(volumeParts string) ([]string, error) { } return strings.Split(volumeParts, ":"), nil } + +func toRef[T any](v T) *T { + return &v +} diff --git a/pipeline/backend/docker/docker.go b/pipeline/backend/docker/docker.go index b592f3c3f2..a2e73e0583 100644 --- a/pipeline/backend/docker/docker.go +++ b/pipeline/backend/docker/docker.go @@ -22,8 +22,9 @@ import ( "net/http" "os" "path/filepath" - "strings" + "time" + "github.com/cenkalti/backoff/v5" "github.com/containerd/errdefs" "github.com/docker/go-connections/tlsconfig" "github.com/moby/moby/api/pkg/stdcopy" @@ -41,7 +42,11 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/shared/utils" ) -var containerKillTimeout = 5 // seconds +const ( + containerKillTimeout = 5 // seconds + volumeRetryWait time.Duration = 1 * time.Second + maxRetry uint = 3 +) type docker struct { client client.APIClient @@ -339,7 +344,7 @@ func (e *docker) DestroyStep(ctx context.Context, step *backend_types.Step, task // we first signal to the container to stop ... if _, err := e.client.ContainerStop(ctx, containerName, client.ContainerStopOptions{ - Timeout: &containerKillTimeout, + Timeout: toRef(containerKillTimeout), }); err != nil && !isErrContainerNotFoundOrNotRunning(err) { // we do not return error yet as we try to kill it first stopErr = fmt.Errorf("could not stop container '%s': %w", step.Name, err) @@ -377,11 +382,24 @@ func (e *docker) DestroyWorkflow(ctx context.Context, conf *backend_types.Config log.Error().Err(err).Msgf("could not destroy all containers") } - if _, err := e.client.VolumeRemove(ctx, conf.Volume, client.VolumeRemoveOptions{ - Force: true, - }); err != nil { + var err error + _, _ = backoff.Retry(ctx, func() (any, error) { + _, err = e.client.VolumeRemove(ctx, conf.Volume, client.VolumeRemoveOptions{ + Force: true, + }) + if err == nil || !isErrVolumeInUse(err) { + // if it worked or if we have no "in use error" do not retry + return nil, nil + } + return nil, err + }, backoff.WithMaxTries(maxRetry), backoff.WithBackOff(&backoff.ExponentialBackOff{ + InitialInterval: volumeRetryWait, + Multiplier: 2, //nolint:mnd + })) + if err != nil { log.Error().Err(err).Msgf("could not remove volume '%s'", conf.Volume) } + if _, err := e.client.NetworkRemove(ctx, conf.Network, client.NetworkRemoveOptions{}); err != nil { log.Error().Err(err).Msgf("could not remove network '%s'", conf.Network) } @@ -394,19 +412,6 @@ var removeOpts = client.ContainerRemoveOptions{ Force: false, } -func isErrContainerNotFoundOrNotRunning(err error) bool { - // Error response from daemon: Cannot kill container: ...: No such container: ... - // Error response from daemon: Cannot kill container: ...: Container ... is not running" - // Error response from podman daemon: can only kill running containers. ... is in state exited - // Error response from daemon: removal of container ... is already in progress - // Error: No such container: ... - return err != nil && - (strings.Contains(err.Error(), "No such container") || - strings.Contains(err.Error(), "is not running") || - strings.Contains(err.Error(), "can only kill running containers") || - (strings.Contains(err.Error(), "removal of container") && strings.Contains(err.Error(), "is already in progress"))) -} - // normalizeArchType converts the arch type reported by docker info into // the runtime.GOARCH format // TODO: find out if we we need to convert other arch types too diff --git a/pipeline/backend/docker/errors.go b/pipeline/backend/docker/errors.go new file mode 100644 index 0000000000..46a7e8463c --- /dev/null +++ b/pipeline/backend/docker/errors.go @@ -0,0 +1,35 @@ +// Copyright 2026 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker + +import "strings" + +func isErrContainerNotFoundOrNotRunning(err error) bool { + // Error response from daemon: Cannot kill container: ...: No such container: ... + // Error response from daemon: Cannot kill container: ...: Container ... is not running" + // Error response from podman daemon: can only kill running containers. ... is in state exited + // Error response from daemon: removal of container ... is already in progress + // Error: No such container: ... + return err != nil && + (strings.Contains(err.Error(), "No such container") || + strings.Contains(err.Error(), "is not running") || + strings.Contains(err.Error(), "can only kill running containers") || + (strings.Contains(err.Error(), "removal of container") && strings.Contains(err.Error(), "is already in progress"))) +} + +func isErrVolumeInUse(err error) bool { + return err != nil && + strings.Contains(err.Error(), "volume is in use") +}