From 2941e508b3bba13c4a7bd7172fb0cc5f018da26b Mon Sep 17 00:00:00 2001 From: Stephen Muth Date: Sat, 3 Jun 2023 18:50:08 -0400 Subject: [PATCH] Allow setting resources for kubernetes on a per-step basis (#1767) This add a simple implementation of requests/limits for individual steps. There is no validation of what the resource actually is beyond checking that it can successfully be converted to a Quantity, so it can be used for things other than just memory/CPU. close #1809 --- .../docs/30-administration/15-agent-config.md | 10 ++- .../22-backends/40-kubernetes.md | 66 ++++++++++++++ docs/docs/30-administration/80-kubernetes.md | 2 +- pipeline/backend/kubernetes/pod.go | 47 ++++------ pipeline/backend/types/backend.go | 6 ++ pipeline/backend/types/backend_kubernetes.go | 12 +++ pipeline/backend/types/step.go | 63 +++++++------- pipeline/frontend/yaml/compiler/convert.go | 71 ++++++++------- pipeline/frontend/yaml/container.go | 87 +++++++++++-------- pipeline/schema/schema.json | 40 +++++++++ 10 files changed, 273 insertions(+), 131 deletions(-) create mode 100644 docs/docs/30-administration/22-backends/40-kubernetes.md create mode 100644 pipeline/backend/types/backend.go create mode 100644 pipeline/backend/types/backend_kubernetes.go diff --git a/docs/docs/30-administration/15-agent-config.md b/docs/docs/30-administration/15-agent-config.md index ac7e15c55..6d1394098 100644 --- a/docs/docs/30-administration/15-agent-config.md +++ b/docs/docs/30-administration/15-agent-config.md @@ -123,15 +123,19 @@ Configures if the gRPC server certificate should be verified, only valid when `W ### `WOODPECKER_BACKEND` > Default: `auto-detect` -Configures the backend engine to run pipelines on. Possible values are `auto-detect`, `docker`, `local` or `ssh`. +Configures the backend engine to run pipelines on. Possible values are `auto-detect`, `docker`, `local`, `ssh` or `kubernetes`. ### `WOODPECKER_BACKEND_DOCKER_*` -See [Docker backend configuration](backends/docker/#configuration) +See [Docker backend configuration](./22-backends/10-docker.md#configuration) ### `WOODPECKER_BACKEND_SSH_*` -See [SSH backend configuration](backends/ssh/#configuration) +See [SSH backend configuration](./22-backends/30-ssh.md#configuration) + +### `WOODPECKER_BACKEND_K8S_*` + +See [Kubernetes backend configuration](./22-backends/40-kubernetes.md#configuration) ## Advanced Settings diff --git a/docs/docs/30-administration/22-backends/40-kubernetes.md b/docs/docs/30-administration/22-backends/40-kubernetes.md new file mode 100644 index 000000000..bb4e7ae43 --- /dev/null +++ b/docs/docs/30-administration/22-backends/40-kubernetes.md @@ -0,0 +1,66 @@ +# Kubernetes backend + +:::caution +Kubernetes support is still experimental and not all pipeline features are fully supported yet. + +Check the [current state](https://github.com/woodpecker-ci/woodpecker/issues/1513) +::: + +The kubernetes backend executes each step inside a newly created pod. A PVC is also created for the lifetime of the pipeline, for transferring files between steps. + +## Configuration + +### `WOODPECKER_BACKEND_K8S_NAMESPACE` +> Default: `woodpecker` + +The namespace to create worker pods in. + +### `WOODPECKER_BACKEND_K8S_VOLUME_SIZE` +> Default: `10G` + +The volume size of the pipeline volume. + +### `WOODPECKER_BACKEND_K8S_STORAGE_CLASS` +> Default: empty + +The storage class to use for the pipeline volume. + +### `WOODPECKER_BACKEND_K8S_STORAGE_RWX` +> Default: `true` + +Determines if `RWX` should be used for the pipeline volume's [access mode](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes). If false, `RWO` is used instead. + +### `WOODPECKER_BACKEND_K8S_POD_LABELS` +> Default: empty + +Additional labels to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-label":"test-value"}`. + +### `WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS` +> Default: empty + +Additional annotations to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-annotation":"test-value"}`. + +## Resources + +The kubernetes backend also allows for specifying requests and limits on a per-step basic, most commonly for CPU and memory. + +Example pipeline configuration: +```yaml +pipeline: + build: + image: golang + commands: + - go get + - go build + - go test + backend_options: + kubernetes: + resources: + requests: + memory: 128Mi + cpu: 1000m + limits: + memory: 256Mi +``` + +See the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for more information on using resources. diff --git a/docs/docs/30-administration/80-kubernetes.md b/docs/docs/30-administration/80-kubernetes.md index 5ba02fbc5..9b14998a4 100644 --- a/docs/docs/30-administration/80-kubernetes.md +++ b/docs/docs/30-administration/80-kubernetes.md @@ -1,6 +1,6 @@ # Kubernetes -Woodpecker does support Kubernetes as a backend. +Woodpecker does support Kubernetes as a backend. See the [Kubernetes backend configuration](./22-backends/40-kubernetes.md#configuration) for backend-specific options. :::caution Kubernetes support is still experimental and not all pipeline features are fully supported yet. diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index b21076c6a..d4e860530 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -1,6 +1,7 @@ package kubernetes import ( + "fmt" "strings" "github.com/woodpecker-ci/woodpecker/pipeline/backend/common" @@ -62,35 +63,21 @@ func Pod(namespace string, step *types.Step, labels, annotations map[string]stri hostAliases = append(hostAliases, v1.HostAlias{IP: host[1], Hostnames: []string{host[0]}}) } - // TODO: add support for resource limits - // if step.Resources.CPULimit == "" { - // step.Resources.CPULimit = "2" - // } - // if step.Resources.MemoryLimit == "" { - // step.Resources.MemoryLimit = "2G" - // } - // memoryLimit := resource.MustParse(step.Resources.MemoryLimit) - // CPULimit := resource.MustParse(step.Resources.CPULimit) - - memoryLimit := resource.MustParse("2G") - CPULimit := resource.MustParse("2") - - memoryLimitValue, _ := memoryLimit.AsInt64() - CPULimitValue, _ := CPULimit.AsInt64() - loadfactor := 0.5 - - memoryRequest := resource.NewQuantity(int64(float64(memoryLimitValue)*loadfactor), resource.DecimalSI) - CPURequest := resource.NewQuantity(int64(float64(CPULimitValue)*loadfactor), resource.DecimalSI) - - resources := v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceMemory: *memoryRequest, - v1.ResourceCPU: *CPURequest, - }, - Limits: v1.ResourceList{ - v1.ResourceMemory: memoryLimit, - v1.ResourceCPU: CPULimit, - }, + resourceRequirements := v1.ResourceRequirements{Requests: v1.ResourceList{}, Limits: v1.ResourceList{}} + var err error + for key, val := range step.BackendOptions.Kubernetes.Resources.Requests { + resourceKey := v1.ResourceName(key) + resourceRequirements.Requests[resourceKey], err = resource.ParseQuantity(val) + if err != nil { + return nil, fmt.Errorf("resource request '%v' quantity '%v': %w", key, val, err) + } + } + for key, val := range step.BackendOptions.Kubernetes.Resources.Limits { + resourceKey := v1.ResourceName(key) + resourceRequirements.Limits[resourceKey], err = resource.ParseQuantity(val) + if err != nil { + return nil, fmt.Errorf("resource limit '%v' quantity '%v': %w", key, val, err) + } } podName, err := dnsName(step.Name) @@ -130,7 +117,7 @@ func Pod(namespace string, step *types.Step, labels, annotations map[string]stri WorkingDir: step.WorkingDir, Env: mapToEnvVars(step.Environment), VolumeMounts: volMounts, - Resources: resources, + Resources: resourceRequirements, SecurityContext: &v1.SecurityContext{ Privileged: &step.Privileged, }, diff --git a/pipeline/backend/types/backend.go b/pipeline/backend/types/backend.go new file mode 100644 index 000000000..89df2c5bd --- /dev/null +++ b/pipeline/backend/types/backend.go @@ -0,0 +1,6 @@ +package types + +// BackendOptions defines advanced options for specific backends +type BackendOptions struct { + Kubernetes KubernetesBackendOptions `json:"kubernetes,omitempty"` +} diff --git a/pipeline/backend/types/backend_kubernetes.go b/pipeline/backend/types/backend_kubernetes.go new file mode 100644 index 000000000..96b71f17e --- /dev/null +++ b/pipeline/backend/types/backend_kubernetes.go @@ -0,0 +1,12 @@ +package types + +// KubernetesBackendOptions defines all the advanced options for the kubernetes backend +type KubernetesBackendOptions struct { + Resources Resources `json:"resouces,omitempty"` +} + +// Resources defines two maps for kubernetes resource definitions +type Resources struct { + Requests map[string]string `json:"requests,omitempty"` + Limits map[string]string `json:"limits,omitempty"` +} diff --git a/pipeline/backend/types/step.go b/pipeline/backend/types/step.go index 419d2d36b..de458ad9b 100644 --- a/pipeline/backend/types/step.go +++ b/pipeline/backend/types/step.go @@ -2,35 +2,36 @@ package types // Step defines a container process. type Step struct { - Name string `json:"name"` - Alias string `json:"alias,omitempty"` - Image string `json:"image,omitempty"` - Pull bool `json:"pull,omitempty"` - Detached bool `json:"detach,omitempty"` - Privileged bool `json:"privileged,omitempty"` - WorkingDir string `json:"working_dir,omitempty"` - Environment map[string]string `json:"environment,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - Entrypoint []string `json:"entrypoint,omitempty"` - Commands []string `json:"commands,omitempty"` - ExtraHosts []string `json:"extra_hosts,omitempty"` - Volumes []string `json:"volumes,omitempty"` - Tmpfs []string `json:"tmpfs,omitempty"` - Devices []string `json:"devices,omitempty"` - Networks []Conn `json:"networks,omitempty"` - DNS []string `json:"dns,omitempty"` - DNSSearch []string `json:"dns_search,omitempty"` - MemSwapLimit int64 `json:"memswap_limit,omitempty"` - MemLimit int64 `json:"mem_limit,omitempty"` - ShmSize int64 `json:"shm_size,omitempty"` - CPUQuota int64 `json:"cpu_quota,omitempty"` - CPUShares int64 `json:"cpu_shares,omitempty"` - CPUSet string `json:"cpu_set,omitempty"` - OnFailure bool `json:"on_failure,omitempty"` - OnSuccess bool `json:"on_success,omitempty"` - Failure string `json:"failure,omitempty"` - AuthConfig Auth `json:"auth_config,omitempty"` - NetworkMode string `json:"network_mode,omitempty"` - IpcMode string `json:"ipc_mode,omitempty"` - Sysctls map[string]string `json:"sysctls,omitempty"` + Name string `json:"name"` + Alias string `json:"alias,omitempty"` + Image string `json:"image,omitempty"` + Pull bool `json:"pull,omitempty"` + Detached bool `json:"detach,omitempty"` + Privileged bool `json:"privileged,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + Environment map[string]string `json:"environment,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Entrypoint []string `json:"entrypoint,omitempty"` + Commands []string `json:"commands,omitempty"` + ExtraHosts []string `json:"extra_hosts,omitempty"` + Volumes []string `json:"volumes,omitempty"` + Tmpfs []string `json:"tmpfs,omitempty"` + Devices []string `json:"devices,omitempty"` + Networks []Conn `json:"networks,omitempty"` + DNS []string `json:"dns,omitempty"` + DNSSearch []string `json:"dns_search,omitempty"` + MemSwapLimit int64 `json:"memswap_limit,omitempty"` + MemLimit int64 `json:"mem_limit,omitempty"` + ShmSize int64 `json:"shm_size,omitempty"` + CPUQuota int64 `json:"cpu_quota,omitempty"` + CPUShares int64 `json:"cpu_shares,omitempty"` + CPUSet string `json:"cpu_set,omitempty"` + OnFailure bool `json:"on_failure,omitempty"` + OnSuccess bool `json:"on_success,omitempty"` + Failure string `json:"failure,omitempty"` + AuthConfig Auth `json:"auth_config,omitempty"` + NetworkMode string `json:"network_mode,omitempty"` + IpcMode string `json:"ipc_mode,omitempty"` + Sysctls map[string]string `json:"sysctls,omitempty"` + BackendOptions BackendOptions `json:"backend_options,omitempty"` } diff --git a/pipeline/frontend/yaml/compiler/convert.go b/pipeline/frontend/yaml/compiler/convert.go index d5e4a2d00..a661f05a4 100644 --- a/pipeline/frontend/yaml/compiler/convert.go +++ b/pipeline/frontend/yaml/compiler/convert.go @@ -110,6 +110,16 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section } } + // Kubernetes advanced settings + backendOptions := backend.BackendOptions{ + Kubernetes: backend.KubernetesBackendOptions{ + Resources: backend.Resources{ + Limits: container.BackendOptions.Kubernetes.Resources.Limits, + Requests: container.BackendOptions.Kubernetes.Resources.Requests, + }, + }, + } + memSwapLimit := int64(container.MemSwapLimit) if c.reslimit.MemSwapLimit != 0 { memSwapLimit = c.reslimit.MemSwapLimit @@ -146,36 +156,37 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section } return &backend.Step{ - Name: name, - Alias: container.Name, - Image: container.Image, - Pull: container.Pull, - Detached: detached, - Privileged: privileged, - WorkingDir: workingdir, - Environment: environment, - Labels: container.Labels, - Commands: container.Commands, - ExtraHosts: container.ExtraHosts, - Volumes: volumes, - Tmpfs: container.Tmpfs, - Devices: container.Devices, - Networks: networks, - DNS: container.DNS, - DNSSearch: container.DNSSearch, - MemSwapLimit: memSwapLimit, - MemLimit: memLimit, - ShmSize: shmSize, - Sysctls: container.Sysctls, - CPUQuota: cpuQuota, - CPUShares: cpuShares, - CPUSet: cpuSet, - AuthConfig: authConfig, - OnSuccess: onSuccess, - OnFailure: onFailure, - Failure: failure, - NetworkMode: networkMode, - IpcMode: ipcMode, + Name: name, + Alias: container.Name, + Image: container.Image, + Pull: container.Pull, + Detached: detached, + Privileged: privileged, + WorkingDir: workingdir, + Environment: environment, + Labels: container.Labels, + Commands: container.Commands, + ExtraHosts: container.ExtraHosts, + Volumes: volumes, + Tmpfs: container.Tmpfs, + Devices: container.Devices, + Networks: networks, + DNS: container.DNS, + DNSSearch: container.DNSSearch, + MemSwapLimit: memSwapLimit, + MemLimit: memLimit, + ShmSize: shmSize, + Sysctls: container.Sysctls, + CPUQuota: cpuQuota, + CPUShares: cpuShares, + CPUSet: cpuSet, + AuthConfig: authConfig, + OnSuccess: onSuccess, + OnFailure: onFailure, + Failure: failure, + NetworkMode: networkMode, + IpcMode: ipcMode, + BackendOptions: backendOptions, } } diff --git a/pipeline/frontend/yaml/container.go b/pipeline/frontend/yaml/container.go index c1e18f18c..bcc835bf4 100644 --- a/pipeline/frontend/yaml/container.go +++ b/pipeline/frontend/yaml/container.go @@ -19,6 +19,20 @@ type ( Email string } + // Advanced backend options + BackendOptions struct { + Kubernetes KubernetesBackendOptions `yaml:"kubernetes,omitempty"` + } + + KubernetesBackendOptions struct { + Resources Resources `yaml:"resources,omitempty"` + } + + Resources struct { + Requests map[string]string `yaml:"requests,omitempty"` + Limits map[string]string `yaml:"limits,omitempty"` + } + // Containers denotes an ordered collection of containers. Containers struct { Containers []*Container @@ -26,42 +40,43 @@ type ( // Container defines a container. Container struct { - AuthConfig AuthConfig `yaml:"auth_config,omitempty"` - CapAdd []string `yaml:"cap_add,omitempty"` - CapDrop []string `yaml:"cap_drop,omitempty"` - Commands types.StringOrSlice `yaml:"commands,omitempty"` - CPUQuota types.StringorInt `yaml:"cpu_quota,omitempty"` - CPUSet string `yaml:"cpuset,omitempty"` - CPUShares types.StringorInt `yaml:"cpu_shares,omitempty"` - Detached bool `yaml:"detach,omitempty"` - Devices []string `yaml:"devices,omitempty"` - Tmpfs []string `yaml:"tmpfs,omitempty"` - DNS types.StringOrSlice `yaml:"dns,omitempty"` - DNSSearch types.StringOrSlice `yaml:"dns_search,omitempty"` - Directory string `yaml:"directory,omitempty"` - Environment types.SliceorMap `yaml:"environment,omitempty"` - ExtraHosts []string `yaml:"extra_hosts,omitempty"` - Group string `yaml:"group,omitempty"` - Image string `yaml:"image,omitempty"` - Failure string `yaml:"failure,omitempty"` - Isolation string `yaml:"isolation,omitempty"` - Labels types.SliceorMap `yaml:"labels,omitempty"` - MemLimit types.MemStringorInt `yaml:"mem_limit,omitempty"` - MemSwapLimit types.MemStringorInt `yaml:"memswap_limit,omitempty"` - MemSwappiness types.MemStringorInt `yaml:"mem_swappiness,omitempty"` - Name string `yaml:"name,omitempty"` - NetworkMode string `yaml:"network_mode,omitempty"` - IpcMode string `yaml:"ipc_mode,omitempty"` - Networks types.Networks `yaml:"networks,omitempty"` - Privileged bool `yaml:"privileged,omitempty"` - Pull bool `yaml:"pull,omitempty"` - ShmSize types.MemStringorInt `yaml:"shm_size,omitempty"` - Ulimits types.Ulimits `yaml:"ulimits,omitempty"` - Volumes types.Volumes `yaml:"volumes,omitempty"` - Secrets Secrets `yaml:"secrets,omitempty"` - Sysctls types.SliceorMap `yaml:"sysctls,omitempty"` - When constraint.When `yaml:"when,omitempty"` - Settings map[string]interface{} `yaml:"settings"` + AuthConfig AuthConfig `yaml:"auth_config,omitempty"` + CapAdd []string `yaml:"cap_add,omitempty"` + CapDrop []string `yaml:"cap_drop,omitempty"` + Commands types.StringOrSlice `yaml:"commands,omitempty"` + CPUQuota types.StringorInt `yaml:"cpu_quota,omitempty"` + CPUSet string `yaml:"cpuset,omitempty"` + CPUShares types.StringorInt `yaml:"cpu_shares,omitempty"` + Detached bool `yaml:"detach,omitempty"` + Devices []string `yaml:"devices,omitempty"` + Tmpfs []string `yaml:"tmpfs,omitempty"` + DNS types.StringOrSlice `yaml:"dns,omitempty"` + DNSSearch types.StringOrSlice `yaml:"dns_search,omitempty"` + Directory string `yaml:"directory,omitempty"` + Environment types.SliceorMap `yaml:"environment,omitempty"` + ExtraHosts []string `yaml:"extra_hosts,omitempty"` + Group string `yaml:"group,omitempty"` + Image string `yaml:"image,omitempty"` + Failure string `yaml:"failure,omitempty"` + Isolation string `yaml:"isolation,omitempty"` + Labels types.SliceorMap `yaml:"labels,omitempty"` + MemLimit types.MemStringorInt `yaml:"mem_limit,omitempty"` + MemSwapLimit types.MemStringorInt `yaml:"memswap_limit,omitempty"` + MemSwappiness types.MemStringorInt `yaml:"mem_swappiness,omitempty"` + Name string `yaml:"name,omitempty"` + NetworkMode string `yaml:"network_mode,omitempty"` + IpcMode string `yaml:"ipc_mode,omitempty"` + Networks types.Networks `yaml:"networks,omitempty"` + Privileged bool `yaml:"privileged,omitempty"` + Pull bool `yaml:"pull,omitempty"` + ShmSize types.MemStringorInt `yaml:"shm_size,omitempty"` + Ulimits types.Ulimits `yaml:"ulimits,omitempty"` + Volumes types.Volumes `yaml:"volumes,omitempty"` + Secrets Secrets `yaml:"secrets,omitempty"` + Sysctls types.SliceorMap `yaml:"sysctls,omitempty"` + When constraint.When `yaml:"when,omitempty"` + Settings map[string]interface{} `yaml:"settings"` + BackendOptions BackendOptions `yaml:"backend_options,omitempty"` } ) diff --git a/pipeline/schema/schema.json b/pipeline/schema/schema.json index 59bb57176..1c0774e8c 100644 --- a/pipeline/schema/schema.json +++ b/pipeline/schema/schema.json @@ -243,6 +243,9 @@ "type": "string", "enum": ["fail", "ignore"], "default": "fail" + }, + "backend_options": { + "$ref": "#/definitions/step_backend_options" } } }, @@ -480,6 +483,43 @@ "description": "Read more: https://woodpecker-ci.org/docs/usage/pipeline-syntax#directory", "type": "string" }, + "step_backend_options": { + "description": "Advanced options for the different agent backends", + "type": "object", + "properties": { + "kubernetes" :{ + "$ref": "#/definitions/step_backend_kubernetes_resources" + } + } + }, + "step_backend_kubernetes": { + "description": "Advanced options for the kubernetes agent backends", + "type": "object", + "properties": { + "resources" :{ + "$ref": "#/definitions/step_backend_kubernetes_resources" + } + } + }, + "step_backend_kubernetes_resources": { + "description": "Resources for the kubernetes backend. Read more: https://woodpecker-ci.org/docs/administration/backends/kubernetes", + "type": "object", + "properties": { + "requests": { + "$ref": "#/definitions/step_kubernetes_resources_object" + }, + "limits": { + "$ref": "#/definitions/step_kubernetes_resources_object" + } + } + }, + "step_kubernetes_resources_object": { + "description": "A list of kubernetes resource mappings", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "services": { "description": "Read more: https://woodpecker-ci.org/docs/usage/services", "type": "object",