diff --git a/docs/docs/30-administration/10-configuration/11-backends/20-kubernetes.md b/docs/docs/30-administration/10-configuration/11-backends/20-kubernetes.md index 416e2730e8..7de7054dab 100644 --- a/docs/docs/30-administration/10-configuration/11-backends/20-kubernetes.md +++ b/docs/docs/30-administration/10-configuration/11-backends/20-kubernetes.md @@ -145,6 +145,100 @@ steps: value: 'value1' effect: 'NoSchedule' tolerationSeconds: 3600 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - eu-central-1a + - eu-central-1b +``` + +### Affinity + +Kubernetes [affinity and anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) rules allow you to constrain which nodes your pods can be scheduled on based on node labels, or co-locate/spread pods relative to other pods. + +You can configure affinity at two levels: + +1. **Per-step via `backend_options.kubernetes.affinity`** (shown in example above) - requires agent configuration to allow it +2. **Agent-wide via `WOODPECKER_BACKEND_K8S_POD_AFFINITY`** - applies to all pods unless overridden + +#### Agent-wide affinity + +To apply affinity rules to all workflow pods, configure the agent with YAML-formatted affinity: + +```yaml +WOODPECKER_BACKEND_K8S_POD_AFFINITY: | + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" +``` + +By default, per-step affinity settings are **not allowed** for security reasons. To enable them: + +```bash +WOODPECKER_BACKEND_K8S_POD_AFFINITY_ALLOW_FROM_STEP: true +``` + +:::warning +Enabling `WOODPECKER_BACKEND_K8S_POD_AFFINITY_ALLOW_FROM_STEP` in multi-tenant environments allows pipeline authors to control pod placement, which may have security or resource isolation implications. +::: + +When per-step affinity is allowed and specified, it **replaces** the agent-wide affinity entirely (not merged). + +#### Example: agent affinity for co-location + +This example configures all workflow pods within a workflow to be co-located on the same node, while requiring other workflows run on different nodes. + +It uses `matchLabelKeys` to dynamically match pods with the same `woodpecker-ci.org/task-uuid`, and `mismatchLabelKeys` to separating pods with different task UUIDs: + +```yaml +WOODPECKER_BACKEND_K8S_POD_AFFINITY: | + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: {} + matchLabelKeys: + - woodpecker-ci.org/task-uuid + topologyKey: "kubernetes.io/hostname" + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: {} + mismatchLabelKeys: + - woodpecker-ci.org/task-uuid + topologyKey: "kubernetes.io/hostname" +``` + +:::note +The `matchLabelKeys` and `mismatchLabelKeys` features require Kubernetes v1.29+ (alpha with feature gate `MatchLabelKeysInPodAffinity`) or v1.33+ (beta, enabled by default). These fields allow the Kubernetes API server to dynamically populate label selectors at pod creation time, eliminating the need to hardcode values like `$(WOODPECKER_TASK_UUID)`. +::: + +#### Example: Node affinity for GPU workloads + +Ensure a step runs only on GPU-enabled nodes: + +```yaml +steps: + - name: train-model + image: tensorflow/tensorflow:latest-gpu + backend_options: + kubernetes: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: accelerator + operator: In + values: + - nvidia-tesla-v100 ``` ### Volumes @@ -153,7 +247,7 @@ To mount volumes a PersistentVolume (PV) and PersistentVolumeClaim (PVC) are nee Persistent volumes must be created manually. Use the Kubernetes [Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) documentation as a reference. -_If your PVC is not highly available or NFS-based, you may also need to integrate affinity settings to ensure that your steps are executed on the correct node._ +_If your PVC is not highly available or NFS-based, use the `affinity` settings (documented above) to ensure that your steps are executed on the correct node._ NOTE: If you plan to use this volume in more than one workflow concurrently, make sure you have configured the PVC in `RWX` mode. Keep in mind that this feature must be supported by the used CSI driver: diff --git a/pipeline/backend/kubernetes/backend_options.go b/pipeline/backend/kubernetes/backend_options.go index 3b228fea31..64fd49f141 100644 --- a/pipeline/backend/kubernetes/backend_options.go +++ b/pipeline/backend/kubernetes/backend_options.go @@ -16,6 +16,7 @@ type BackendOptions struct { Annotations map[string]string `mapstructure:"annotations"` NodeSelector map[string]string `mapstructure:"nodeSelector"` Tolerations []Toleration `mapstructure:"tolerations"` + Affinity *v1.Affinity `mapstructure:"affinity"` SecurityContext *SecurityContext `mapstructure:"securityContext"` Secrets []SecretRef `mapstructure:"secrets"` } diff --git a/pipeline/backend/kubernetes/backend_options_test.go b/pipeline/backend/kubernetes/backend_options_test.go index ff44202c81..14526a8f85 100644 --- a/pipeline/backend/kubernetes/backend_options_test.go +++ b/pipeline/backend/kubernetes/backend_options_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" ) @@ -37,6 +39,19 @@ func Test_parseBackendOptions(t *testing.T) { "tolerations": []map[string]any{ {"key": "net-port", "value": "100Mbit", "effect": TaintEffectNoSchedule}, }, + "affinity": map[string]any{ + "podAffinity": map[string]any{ + "requiredDuringSchedulingIgnoredDuringExecution": []map[string]any{ + { + "labelSelector": map[string]any{}, + "matchLabelKeys": []string{ + "woodpecker-ci.org/task-uuid", + }, + "topologyKey": "kubernetes.io/hostname", + }, + }, + }, + }, "resources": map[string]any{ "requests": map[string]string{"memory": "128Mi", "cpu": "1000m"}, "limits": map[string]string{"memory": "256Mi", "cpu": "2"}, @@ -81,6 +96,19 @@ func Test_parseBackendOptions(t *testing.T) { Labels: map[string]string{"app": "test"}, Annotations: map[string]string{"apps.kubernetes.io/pod-index": "0"}, Tolerations: []Toleration{{Key: "net-port", Value: "100Mbit", Effect: TaintEffectNoSchedule}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{}, + MatchLabelKeys: []string{ + "woodpecker-ci.org/task-uuid", + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, Resources: Resources{ Requests: map[string]string{"memory": "128Mi", "cpu": "1000m"}, Limits: map[string]string{"memory": "256Mi", "cpu": "2"}, diff --git a/pipeline/backend/kubernetes/flags.go b/pipeline/backend/kubernetes/flags.go index f6b290fdc6..fdec8e496d 100644 --- a/pipeline/backend/kubernetes/flags.go +++ b/pipeline/backend/kubernetes/flags.go @@ -91,6 +91,18 @@ var Flags = []cli.Flag{ Usage: "whether to allow using tolerations from step's backend options", Value: true, }, + &cli.StringFlag{ + Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_POD_AFFINITY"), + Name: "backend-k8s-pod-affinity", + Usage: "backend k8s Agent-wide worker pod affinity, in YAML format", + Value: "", + }, + &cli.BoolFlag{ + Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_POD_AFFINITY_ALLOW_FROM_STEP"), + Name: "backend-k8s-pod-affinity-allow-from-step", + Usage: "whether to allow using affinity from step's backend options", + Value: false, + }, &cli.BoolFlag{ Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_SECCTX_NONROOT"), // cspell:words secctx nonroot Name: "backend-k8s-secctx-nonroot", diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index 02d2ab869d..49448cede9 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -30,7 +30,6 @@ import ( backoff "github.com/cenkalti/backoff/v5" "github.com/rs/zerolog/log" "github.com/urfave/cli/v3" - "gopkg.in/yaml.v3" v1 "k8s.io/api/core/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" @@ -39,6 +38,7 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // To authenticate to GCP K8s clusters "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" + "sigs.k8s.io/yaml" "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" ) @@ -71,6 +71,8 @@ type config struct { PodNodeSelector map[string]string PodTolerationsAllowFromStep bool PodTolerations []Toleration + PodAffinity *v1.Affinity + PodAffinityAllowFromStep bool ImagePullSecretNames []string SecurityContext SecurityContextConfig NativeSecretsAllowFromStep bool @@ -115,6 +117,7 @@ func configFromCliContext(ctx context.Context) (*config, error) { PodAnnotationsAllowFromStep: c.Bool("backend-k8s-pod-annotations-allow-from-step"), PodTolerationsAllowFromStep: c.Bool("backend-k8s-pod-tolerations-allow-from-step"), PodNodeSelector: make(map[string]string), // just init empty map to prevent nil panic + PodAffinityAllowFromStep: c.Bool("backend-k8s-pod-affinity-allow-from-step"), ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"), SecurityContext: SecurityContextConfig{ RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"), // cspell:words secctx nonroot @@ -147,6 +150,12 @@ func configFromCliContext(ctx context.Context) (*config, error) { return nil, err } } + if podAffinity := c.String("backend-k8s-pod-affinity"); podAffinity != "" { + if err := yaml.Unmarshal([]byte(podAffinity), &config.PodAffinity); err != nil { + log.Error().Err(err).Msgf("could not unmarshal pod affinity '%s'", podAffinity) + return nil, err + } + } return &config, nil } diff --git a/pipeline/backend/kubernetes/kubernetes_test.go b/pipeline/backend/kubernetes/kubernetes_test.go index c5c2902bd1..e911bd2477 100644 --- a/pipeline/backend/kubernetes/kubernetes_test.go +++ b/pipeline/backend/kubernetes/kubernetes_test.go @@ -19,6 +19,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v3" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -130,3 +132,46 @@ func TestSetupWorkflow(t *testing.T) { _, err = engine.client.CoreV1().Services(namespace).Get(context.Background(), "wp-hsvc-"+taskUUID, meta_v1.GetOptions{}) assert.NoError(t, err, "headless service should be created during workflow setup") } + +func TestAffinityFromCliContext(t *testing.T) { + t.Setenv("WOODPECKER_BACKEND_K8S_NAMESPACE", "") + t.Setenv("WOODPECKER_BACKEND_K8S_POD_AFFINITY", `{ + "podAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": [ + { + "labelSelector": {}, + "matchLabelKeys": [ + "woodpecker-ci.org/task-uuid" + ], + "topologyKey": "kubernetes.io/hostname" + } + ] + } + }`) + t.Setenv("WOODPECKER_BACKEND_K8S_POD_AFFINITY_ALLOW_FROM_STEP", "false") + + cmd := &cli.Command{ + Flags: Flags, + Action: func(ctx context.Context, c *cli.Command) error { + ctx = context.WithValue(ctx, types.CliCommand, c) + config, err := configFromCliContext(ctx) + + require.NoError(t, err) + require.NotNil(t, config) + assert.False(t, config.PodAffinityAllowFromStep) + + // Verify affinity was parsed + require.NotNil(t, config.PodAffinity) + require.NotNil(t, config.PodAffinity.PodAffinity) + require.Len(t, config.PodAffinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 1) + + term := config.PodAffinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0] + assert.Equal(t, "kubernetes.io/hostname", term.TopologyKey) + assert.Equal(t, []string{"woodpecker-ci.org/task-uuid"}, term.MatchLabelKeys) + + return nil + }, + } + err := cmd.Run(context.Background(), []string{"test"}) + require.NoError(t, err) +} diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index d1bfd92e99..04b3373afa 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -189,6 +189,7 @@ func podSpec(step *types.Step, config *config, options BackendOptions, nsp nativ DNSConfig: dnsConfig(config.GetNamespace(step.OrgID), subdomain), NodeSelector: nodeSelector(options.NodeSelector, config.PodNodeSelector, step.Environment["CI_SYSTEM_PLATFORM"]), Tolerations: tolerations(options.Tolerations), + Affinity: affinity(options.Affinity, config.PodAffinity, config.PodAffinityAllowFromStep), SecurityContext: podSecurityContext(options.SecurityContext, config.SecurityContext, step.Privileged), } @@ -472,6 +473,25 @@ func toleration(backendToleration Toleration) v1.Toleration { } } +func affinity(stepAffinity, agentAffinity *v1.Affinity, allowFromStep bool) *v1.Affinity { + if stepAffinity != nil { + if allowFromStep { + log.Trace().Msg("using affinity from step backend options") + return stepAffinity + } else { + log.Debug().Msg("Step affinity is disallowed by instance configuration, ignoring it") + } + } + + if agentAffinity != nil { + log.Trace().Msg("using affinity from agent configuration") + return agentAffinity + } + + log.Trace().Msg("no affinity configured") + return nil +} + func podSecurityContext(sc *SecurityContext, secCtxConf SecurityContextConfig, stepPrivileged bool) *v1.PodSecurityContext { var ( nonRoot *bool diff --git a/pipeline/backend/kubernetes/pod_test.go b/pipeline/backend/kubernetes/pod_test.go index 1ff984c8ad..ec528bb373 100644 --- a/pipeline/backend/kubernetes/pod_test.go +++ b/pipeline/backend/kubernetes/pod_test.go @@ -21,6 +21,7 @@ import ( "github.com/kinbiko/jsonassert" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" ) @@ -904,3 +905,307 @@ func TestStepSecret(t *testing.T) { ja := jsonassert.New(t) ja.Assertf(string(secretJSON), expected) } + +func TestPodAffinity(t *testing.T) { + const expected = ` + { + "metadata": { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "namespace": "woodpecker", + "labels": { + "step": "affinity-test", + "woodpecker-ci.org/step": "affinity-test", + "woodpecker-ci.org/task-uuid": "11301" + } + }, + "spec": { + "containers": [ + { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "image": "alpine", + "resources": {} + } + ], + "restartPolicy": "Never", + "affinity": { + "podAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": [ + { + "labelSelector": {}, + "matchLabelKeys": ["woodpecker-ci.org/task-uuid"], + "topologyKey": "kubernetes.io/hostname" + } + ] + } + }, + "dnsConfig": { + "searches": ["wp-hsvc-11301.woodpecker.svc.cluster.local"] + }, + "subdomain": "wp-hsvc-11301", + "hostname": "affinity-test" + }, + "status": {} + }` + + agentAffinity := &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &meta_v1.LabelSelector{}, + MatchLabelKeys: []string{"woodpecker-ci.org/task-uuid"}, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + } + + pod, err := mkPod(&types.Step{ + Name: "affinity-test", + Image: "alpine", + UUID: "01he8bebctabr3kgk0qj36d2me-0", + }, &config{ + Namespace: "woodpecker", + PodAffinity: agentAffinity, + PodAffinityAllowFromStep: false, + }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{}, taskUUID) + assert.NoError(t, err) + + podJSON, err := json.Marshal(pod) + assert.NoError(t, err) + + ja := jsonassert.New(t) + ja.Assertf(string(podJSON), expected) +} + +func TestPodAffinityAllowFromStep(t *testing.T) { + const expectedDisallow = ` + { + "metadata": { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "namespace": "woodpecker", + "labels": { + "step": "affinity-test", + "woodpecker-ci.org/step": "affinity-test", + "woodpecker-ci.org/task-uuid": "11301" + } + }, + "spec": { + "containers": [ + { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "image": "alpine", + "resources": {} + } + ], + "restartPolicy": "Never", + "dnsConfig": { + "searches": ["wp-hsvc-11301.woodpecker.svc.cluster.local"] + }, + "subdomain": "wp-hsvc-11301", + "hostname": "affinity-test" + }, + "status": {} + }` + const expectedAllow = ` + { + "metadata": { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "namespace": "woodpecker", + "labels": { + "step": "affinity-test", + "woodpecker-ci.org/step": "affinity-test", + "woodpecker-ci.org/task-uuid": "11301" + } + }, + "spec": { + "containers": [ + { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "image": "alpine", + "resources": {} + } + ], + "restartPolicy": "Never", + "affinity": { + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "weight": 100, + "podAffinityTerm": { + "labelSelector": { + "matchLabels": { + "app": "woodpecker" + } + }, + "topologyKey": "kubernetes.io/hostname" + } + } + ] + } + }, + "dnsConfig": { + "searches": ["wp-hsvc-11301.woodpecker.svc.cluster.local"] + }, + "subdomain": "wp-hsvc-11301", + "hostname": "affinity-test" + }, + "status": {} + }` + + stepAffinity := &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &meta_v1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "woodpecker", + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + } + + step := &types.Step{ + Name: "affinity-test", + Image: "alpine", + UUID: "01he8bebctabr3kgk0qj36d2me-0", + } + + pod, err := mkPod(step, &config{ + Namespace: "woodpecker", + PodAffinity: nil, + PodAffinityAllowFromStep: false, + }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ + Affinity: stepAffinity, + }, taskUUID) + assert.NoError(t, err) + + podJSON, err := json.Marshal(pod) + assert.NoError(t, err) + + ja := jsonassert.New(t) + ja.Assertf(string(podJSON), expectedDisallow) + + pod, err = mkPod(step, &config{ + Namespace: "woodpecker", + PodAffinity: nil, + PodAffinityAllowFromStep: true, + }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ + Affinity: stepAffinity, + }, taskUUID) + assert.NoError(t, err) + + podJSON, err = json.Marshal(pod) + assert.NoError(t, err) + + ja = jsonassert.New(t) + ja.Assertf(string(podJSON), expectedAllow) +} + +func TestPodAffinityStepOverridesAgent(t *testing.T) { + const expected = ` + { + "metadata": { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "namespace": "woodpecker", + "labels": { + "step": "affinity-test", + "woodpecker-ci.org/step": "affinity-test", + "woodpecker-ci.org/task-uuid": "11301" + } + }, + "spec": { + "containers": [ + { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "image": "alpine", + "resources": {} + } + ], + "restartPolicy": "Never", + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "disk-type", + "operator": "In", + "values": ["ssd"] + } + ] + } + ] + } + } + }, + "dnsConfig": { + "searches": ["wp-hsvc-11301.woodpecker.svc.cluster.local"] + }, + "subdomain": "wp-hsvc-11301", + "hostname": "affinity-test" + }, + "status": {} + }` + + agentAffinity := &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: v1.NodeSelectorOpIn, + Values: []string{"eu-central-1a"}, + }, + }, + }, + }, + }, + }, + } + + stepAffinity := &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "disk-type", + Operator: v1.NodeSelectorOpIn, + Values: []string{"ssd"}, + }, + }, + }, + }, + }, + }, + } + + pod, err := mkPod(&types.Step{ + Name: "affinity-test", + Image: "alpine", + UUID: "01he8bebctabr3kgk0qj36d2me-0", + }, &config{ + Namespace: "woodpecker", + PodAffinity: agentAffinity, + PodAffinityAllowFromStep: true, + }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ + Affinity: stepAffinity, + }, taskUUID) + assert.NoError(t, err) + + podJSON, err := json.Marshal(pod) + assert.NoError(t, err) + + ja := jsonassert.New(t) + ja.Assertf(string(podJSON), expected) +}