self-hosted/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2026-06-03 16:35:37 +02:00
Code Issues Releases Activity

feat(kubernetes): add support for pod affinity and anti-affinity configurations (#5854)

Browse Source
This commit is contained in:
Martin Schmidt
2025-12-17 13:59:22 +01:00
committed by GitHub
parent 971ab734af
commit b7b33db98b
8 changed files with 516 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/docs/30-administration/10-configuration/11-backends/20-kubernetes.md
+95 -1
View File
@@ -145,6 +145,100 @@ steps:
value: 'value1'
effect: 'NoSchedule'
tolerationSeconds: 3600
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-central-1a
- eu-central-1b
```
### Affinity
Kubernetes [affinity and anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) rules allow you to constrain which nodes your pods can be scheduled on based on node labels, or co-locate/spread pods relative to other pods.
You can configure affinity at two levels:
1. **Per-step via `backend_options.kubernetes.affinity`** (shown in example above) - requires agent configuration to allow it
2. **Agent-wide via `WOODPECKER_BACKEND_K8S_POD_AFFINITY`** - applies to all pods unless overridden
#### Agent-wide affinity
To apply affinity rules to all workflow pods, configure the agent with YAML-formatted affinity:
```yaml
WOODPECKER_BACKEND_K8S_POD_AFFINITY: |
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/worker
operator: In
values:
- "true"
```
By default, per-step affinity settings are **not allowed** for security reasons. To enable them:
```bash
WOODPECKER_BACKEND_K8S_POD_AFFINITY_ALLOW_FROM_STEP: true
```
:::warning
Enabling `WOODPECKER_BACKEND_K8S_POD_AFFINITY_ALLOW_FROM_STEP` in multi-tenant environments allows pipeline authors to control pod placement, which may have security or resource isolation implications.
:::
When per-step affinity is allowed and specified, it **replaces** the agent-wide affinity entirely (not merged).
#### Example: agent affinity for co-location
This example configures all workflow pods within a workflow to be co-located on the same node, while requiring other workflows run on different nodes.
It uses `matchLabelKeys` to dynamically match pods with the same `woodpecker-ci.org/task-uuid`, and `mismatchLabelKeys` to separating pods with different task UUIDs:
```yaml
WOODPECKER_BACKEND_K8S_POD_AFFINITY: |
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector: {}
matchLabelKeys:
- woodpecker-ci.org/task-uuid
topologyKey: "kubernetes.io/hostname"
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector: {}
mismatchLabelKeys:
- woodpecker-ci.org/task-uuid
topologyKey: "kubernetes.io/hostname"
```
:::note
The `matchLabelKeys` and `mismatchLabelKeys` features require Kubernetes v1.29+ (alpha with feature gate `MatchLabelKeysInPodAffinity`) or v1.33+ (beta, enabled by default). These fields allow the Kubernetes API server to dynamically populate label selectors at pod creation time, eliminating the need to hardcode values like `$(WOODPECKER_TASK_UUID)`.
:::
#### Example: Node affinity for GPU workloads
Ensure a step runs only on GPU-enabled nodes:
```yaml
steps:
- name: train-model
image: tensorflow/tensorflow:latest-gpu
backend_options:
kubernetes:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: accelerator
operator: In
values:
- nvidia-tesla-v100
```
### Volumes
@@ -153,7 +247,7 @@ To mount volumes a PersistentVolume (PV) and PersistentVolumeClaim (PVC) are nee
Persistent volumes must be created manually. Use the Kubernetes [Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) documentation as a reference.
_If your PVC is not highly available or NFS-based, you may also need to integrate affinity settings to ensure that your steps are executed on the correct node._
_If your PVC is not highly available or NFS-based, use the `affinity` settings (documented above) to ensure that your steps are executed on the correct node._
NOTE: If you plan to use this volume in more than one workflow concurrently, make sure you have configured the PVC in `RWX` mode. Keep in mind that this feature must be supported by the used CSI driver:
pipeline/backend/kubernetes/backend_options.go
+1
View File
@@ -16,6 +16,7 @@ type BackendOptions struct {
Annotations map[string]string `mapstructure:"annotations"`
NodeSelector map[string]string `mapstructure:"nodeSelector"`
Tolerations []Toleration `mapstructure:"tolerations"`
Affinity *v1.Affinity `mapstructure:"affinity"`
SecurityContext *SecurityContext `mapstructure:"securityContext"`
Secrets []SecretRef `mapstructure:"secrets"`
}
pipeline/backend/kubernetes/backend_options_test.go
+28
View File
@@ -4,6 +4,8 @@ import (
"testing"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types"
)
@@ -37,6 +39,19 @@ func Test_parseBackendOptions(t *testing.T) {
"tolerations": []map[string]any{
{"key": "net-port", "value": "100Mbit", "effect": TaintEffectNoSchedule},
},
"affinity": map[string]any{
"podAffinity": map[string]any{
"requiredDuringSchedulingIgnoredDuringExecution": []map[string]any{
{
"labelSelector": map[string]any{},
"matchLabelKeys": []string{
"woodpecker-ci.org/task-uuid",
},
"topologyKey": "kubernetes.io/hostname",
},
},
},
},
"resources": map[string]any{
"requests": map[string]string{"memory": "128Mi", "cpu": "1000m"},
"limits": map[string]string{"memory": "256Mi", "cpu": "2"},
@@ -81,6 +96,19 @@ func Test_parseBackendOptions(t *testing.T) {
Labels: map[string]string{"app": "test"},
Annotations: map[string]string{"apps.kubernetes.io/pod-index": "0"},
Tolerations: []Toleration{{Key: "net-port", Value: "100Mbit", Effect: TaintEffectNoSchedule}},
Affinity: &v1.Affinity{
PodAffinity: &v1.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
{
LabelSelector: &metav1.LabelSelector{},
MatchLabelKeys: []string{
"woodpecker-ci.org/task-uuid",
},
TopologyKey: "kubernetes.io/hostname",
},
},
},
},
Resources: Resources{
Requests: map[string]string{"memory": "128Mi", "cpu": "1000m"},
Limits: map[string]string{"memory": "256Mi", "cpu": "2"},
pipeline/backend/kubernetes/flags.go
+12
View File
@@ -91,6 +91,18 @@ var Flags = []cli.Flag{
Usage: "whether to allow using tolerations from step's backend options",
Value: true,
},
&cli.StringFlag{
Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_POD_AFFINITY"),
Name: "backend-k8s-pod-affinity",
Usage: "backend k8s Agent-wide worker pod affinity, in YAML format",
Value: "",
},
&cli.BoolFlag{
Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_POD_AFFINITY_ALLOW_FROM_STEP"),
Name: "backend-k8s-pod-affinity-allow-from-step",
Usage: "whether to allow using affinity from step's backend options",
Value: false,
},
&cli.BoolFlag{
Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_SECCTX_NONROOT"), // cspell:words secctx nonroot
Name: "backend-k8s-secctx-nonroot",
pipeline/backend/kubernetes/kubernetes.go
+10 -1
View File
@@ -30,7 +30,6 @@ import (
backoff "github.com/cenkalti/backoff/v5"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v3"
"gopkg.in/yaml.v3"
v1 "k8s.io/api/core/v1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/informers"
@@ -39,6 +38,7 @@ import (
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // To authenticate to GCP K8s clusters
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/yaml"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types"
)
@@ -71,6 +71,8 @@ type config struct {
PodNodeSelector map[string]string
PodTolerationsAllowFromStep bool
PodTolerations []Toleration
PodAffinity *v1.Affinity
PodAffinityAllowFromStep bool
ImagePullSecretNames []string
SecurityContext SecurityContextConfig
NativeSecretsAllowFromStep bool
@@ -115,6 +117,7 @@ func configFromCliContext(ctx context.Context) (*config, error) {
PodAnnotationsAllowFromStep: c.Bool("backend-k8s-pod-annotations-allow-from-step"),
PodTolerationsAllowFromStep: c.Bool("backend-k8s-pod-tolerations-allow-from-step"),
PodNodeSelector: make(map[string]string), // just init empty map to prevent nil panic
PodAffinityAllowFromStep: c.Bool("backend-k8s-pod-affinity-allow-from-step"),
ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"),
SecurityContext: SecurityContextConfig{
RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"), // cspell:words secctx nonroot
@@ -147,6 +150,12 @@ func configFromCliContext(ctx context.Context) (*config, error) {
return nil, err
}
}
if podAffinity := c.String("backend-k8s-pod-affinity"); podAffinity != "" {
if err := yaml.Unmarshal([]byte(podAffinity), &config.PodAffinity); err != nil {
log.Error().Err(err).Msgf("could not unmarshal pod affinity '%s'", podAffinity)
return nil, err
}
}
return &config, nil
}
pipeline/backend/kubernetes/kubernetes_test.go
+45
View File
@@ -19,6 +19,8 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v3"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
@@ -130,3 +132,46 @@ func TestSetupWorkflow(t *testing.T) {
_, err = engine.client.CoreV1().Services(namespace).Get(context.Background(), "wp-hsvc-"+taskUUID, meta_v1.GetOptions{})
assert.NoError(t, err, "headless service should be created during workflow setup")
}
func TestAffinityFromCliContext(t *testing.T) {
t.Setenv("WOODPECKER_BACKEND_K8S_NAMESPACE", "")
t.Setenv("WOODPECKER_BACKEND_K8S_POD_AFFINITY", `{
"podAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": [
{
"labelSelector": {},
"matchLabelKeys": [
"woodpecker-ci.org/task-uuid"
],
"topologyKey": "kubernetes.io/hostname"
}
]
}
}`)
t.Setenv("WOODPECKER_BACKEND_K8S_POD_AFFINITY_ALLOW_FROM_STEP", "false")
cmd := &cli.Command{
Flags: Flags,
Action: func(ctx context.Context, c *cli.Command) error {
ctx = context.WithValue(ctx, types.CliCommand, c)
config, err := configFromCliContext(ctx)
require.NoError(t, err)
require.NotNil(t, config)
assert.False(t, config.PodAffinityAllowFromStep)
// Verify affinity was parsed
require.NotNil(t, config.PodAffinity)
require.NotNil(t, config.PodAffinity.PodAffinity)
require.Len(t, config.PodAffinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 1)
term := config.PodAffinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0]
assert.Equal(t, "kubernetes.io/hostname", term.TopologyKey)
assert.Equal(t, []string{"woodpecker-ci.org/task-uuid"}, term.MatchLabelKeys)
return nil
},
}
err := cmd.Run(context.Background(), []string{"test"})
require.NoError(t, err)
}
pipeline/backend/kubernetes/pod.go
+20
View File
@@ -189,6 +189,7 @@ func podSpec(step *types.Step, config *config, options BackendOptions, nsp nativ
DNSConfig: dnsConfig(config.GetNamespace(step.OrgID), subdomain),
NodeSelector: nodeSelector(options.NodeSelector, config.PodNodeSelector, step.Environment["CI_SYSTEM_PLATFORM"]),
Tolerations: tolerations(options.Tolerations),
Affinity: affinity(options.Affinity, config.PodAffinity, config.PodAffinityAllowFromStep),
SecurityContext: podSecurityContext(options.SecurityContext, config.SecurityContext, step.Privileged),
}
@@ -472,6 +473,25 @@ func toleration(backendToleration Toleration) v1.Toleration {
}
}
func affinity(stepAffinity, agentAffinity *v1.Affinity, allowFromStep bool) *v1.Affinity {
if stepAffinity != nil {
if allowFromStep {
log.Trace().Msg("using affinity from step backend options")
return stepAffinity
} else {
log.Debug().Msg("Step affinity is disallowed by instance configuration, ignoring it")
}
}
if agentAffinity != nil {
log.Trace().Msg("using affinity from agent configuration")
return agentAffinity
}
log.Trace().Msg("no affinity configured")
return nil
}
func podSecurityContext(sc *SecurityContext, secCtxConf SecurityContextConfig, stepPrivileged bool) *v1.PodSecurityContext {
var (
nonRoot *bool
pipeline/backend/kubernetes/pod_test.go
+305
View File
@@ -21,6 +21,7 @@ import (
"github.com/kinbiko/jsonassert"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types"
)
@@ -904,3 +905,307 @@ func TestStepSecret(t *testing.T) {
ja := jsonassert.New(t)
ja.Assertf(string(secretJSON), expected)
}
func TestPodAffinity(t *testing.T) {
const expected = `
{
"metadata": {
"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
"namespace": "woodpecker",
"labels": {
"step": "affinity-test",
"woodpecker-ci.org/step": "affinity-test",
"woodpecker-ci.org/task-uuid": "11301"
}
},
"spec": {
"containers": [
{
"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
"image": "alpine",
"resources": {}
}
],
"restartPolicy": "Never",
"affinity": {
"podAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": [
{
"labelSelector": {},
"matchLabelKeys": ["woodpecker-ci.org/task-uuid"],
"topologyKey": "kubernetes.io/hostname"
}
]
}
},
"dnsConfig": {
"searches": ["wp-hsvc-11301.woodpecker.svc.cluster.local"]
},
"subdomain": "wp-hsvc-11301",
"hostname": "affinity-test"
},
"status": {}
}`
agentAffinity := &v1.Affinity{
PodAffinity: &v1.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
{
LabelSelector: &meta_v1.LabelSelector{},
MatchLabelKeys: []string{"woodpecker-ci.org/task-uuid"},
TopologyKey: "kubernetes.io/hostname",
},
},
},
}
pod, err := mkPod(&types.Step{
Name: "affinity-test",
Image: "alpine",
UUID: "01he8bebctabr3kgk0qj36d2me-0",
}, &config{
Namespace: "woodpecker",
PodAffinity: agentAffinity,
PodAffinityAllowFromStep: false,
}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{}, taskUUID)
assert.NoError(t, err)
podJSON, err := json.Marshal(pod)
assert.NoError(t, err)
ja := jsonassert.New(t)
ja.Assertf(string(podJSON), expected)
}
func TestPodAffinityAllowFromStep(t *testing.T) {
const expectedDisallow = `
{
"metadata": {
"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
"namespace": "woodpecker",
"labels": {
"step": "affinity-test",
"woodpecker-ci.org/step": "affinity-test",
"woodpecker-ci.org/task-uuid": "11301"
}
},
"spec": {
"containers": [
{
"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
"image": "alpine",
"resources": {}
}
],
"restartPolicy": "Never",
"dnsConfig": {
"searches": ["wp-hsvc-11301.woodpecker.svc.cluster.local"]
},
"subdomain": "wp-hsvc-11301",
"hostname": "affinity-test"
},
"status": {}
}`
const expectedAllow = `
{
"metadata": {
"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
"namespace": "woodpecker",
"labels": {
"step": "affinity-test",
"woodpecker-ci.org/step": "affinity-test",
"woodpecker-ci.org/task-uuid": "11301"
}
},
"spec": {
"containers": [
{
"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
"image": "alpine",
"resources": {}
}
],
"restartPolicy": "Never",
"affinity": {
"podAntiAffinity": {
"preferredDuringSchedulingIgnoredDuringExecution": [
{
"weight": 100,
"podAffinityTerm": {
"labelSelector": {
"matchLabels": {
"app": "woodpecker"
}
},
"topologyKey": "kubernetes.io/hostname"
}
}
]
}
},
"dnsConfig": {
"searches": ["wp-hsvc-11301.woodpecker.svc.cluster.local"]
},
"subdomain": "wp-hsvc-11301",
"hostname": "affinity-test"
},
"status": {}
}`
stepAffinity := &v1.Affinity{
PodAntiAffinity: &v1.PodAntiAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
{
Weight: 100,
PodAffinityTerm: v1.PodAffinityTerm{
LabelSelector: &meta_v1.LabelSelector{
MatchLabels: map[string]string{
"app": "woodpecker",
},
},
TopologyKey: "kubernetes.io/hostname",
},
},
},
},
}
step := &types.Step{
Name: "affinity-test",
Image: "alpine",
UUID: "01he8bebctabr3kgk0qj36d2me-0",
}
pod, err := mkPod(step, &config{
Namespace: "woodpecker",
PodAffinity: nil,
PodAffinityAllowFromStep: false,
}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{
Affinity: stepAffinity,
}, taskUUID)
assert.NoError(t, err)
podJSON, err := json.Marshal(pod)
assert.NoError(t, err)
ja := jsonassert.New(t)
ja.Assertf(string(podJSON), expectedDisallow)
pod, err = mkPod(step, &config{
Namespace: "woodpecker",
PodAffinity: nil,
PodAffinityAllowFromStep: true,
}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{
Affinity: stepAffinity,
}, taskUUID)
assert.NoError(t, err)
podJSON, err = json.Marshal(pod)
assert.NoError(t, err)
ja = jsonassert.New(t)
ja.Assertf(string(podJSON), expectedAllow)
}
func TestPodAffinityStepOverridesAgent(t *testing.T) {
const expected = `
{
"metadata": {
"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
"namespace": "woodpecker",
"labels": {
"step": "affinity-test",
"woodpecker-ci.org/step": "affinity-test",
"woodpecker-ci.org/task-uuid": "11301"
}
},
"spec": {
"containers": [
{
"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
"image": "alpine",
"resources": {}
}
],
"restartPolicy": "Never",
"affinity": {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "disk-type",
"operator": "In",
"values": ["ssd"]
}
]
}
]
}
}
},
"dnsConfig": {
"searches": ["wp-hsvc-11301.woodpecker.svc.cluster.local"]
},
"subdomain": "wp-hsvc-11301",
"hostname": "affinity-test"
},
"status": {}
}`
agentAffinity := &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "topology.kubernetes.io/zone",
Operator: v1.NodeSelectorOpIn,
Values: []string{"eu-central-1a"},
},
},
},
},
},
},
}
stepAffinity := &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "disk-type",
Operator: v1.NodeSelectorOpIn,
Values: []string{"ssd"},
},
},
},
},
},
},
}
pod, err := mkPod(&types.Step{
Name: "affinity-test",
Image: "alpine",
UUID: "01he8bebctabr3kgk0qj36d2me-0",
}, &config{
Namespace: "woodpecker",
PodAffinity: agentAffinity,
PodAffinityAllowFromStep: true,
}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{
Affinity: stepAffinity,
}, taskUUID)
assert.NoError(t, err)
podJSON, err := json.Marshal(pod)
assert.NoError(t, err)
ja := jsonassert.New(t)
ja.Assertf(string(podJSON), expected)
}