self-hosted/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-12-24 10:07:21 +02:00
Code Issues Releases Activity

Add user as docker backend_option (#4526)

Browse Source
This commit is contained in:
Robert Kaussow 2024-12-08 12:02:35 +01:00 committed by GitHub
parent d35d950c44
commit 786a8fb003
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 226 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
docs/docs/30-administration/22-backends/10-docker.md
View File

@ -18,6 +18,25 @@ FROM woodpeckerci/woodpecker-server:latest-alpine
RUN apk add -U --no-cache docker-credential-ecr-login
```
## Step specific configuration
### Run user
By default the docker backend starts the step container without the `--user` flag. This means the step container will use the default user of the container. To change this behavior you can set the `user` backend option to the preferred user/group:
```yaml
steps:
- name: example
image: alpine
commands:
- whoami
backend_options:
docker:
user: 65534:65534
```
The syntax is the same as the [docker run](https://docs.docker.com/engine/reference/run/#user) `--user` flag.
## Image cleanup
The agent **will not** automatically remove images from the host. This task should be managed by the host system. For example, you can use a cron job to periodically do clean-up tasks for the CI runner.

4
docs/docs/30-administration/22-backends/40-kubernetes.md
View File

@ -12,7 +12,7 @@ In addition to [registries specified in the UI](../../20-usage/41-registries.md)
Place these Secrets in namespace defined by `WOODPECKER_BACKEND_K8S_NAMESPACE` and provide the Secret names to Agents via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`.
## Job specific configuration
## Step specific configuration
### Resources
@ -67,7 +67,7 @@ To give steps access to the Kubernetes API via service account, take a look at [
### Node selector
`nodeSelector` specifies the labels which are used to select the node on which the job will be executed.
`nodeSelector` specifies the labels which are used to select the node on which the step will be executed.
Labels defined here will be appended to a list which already contains `"kubernetes.io/arch"`.
By default `"kubernetes.io/arch"` is inferred from the agents' platform. One can override it by setting that label in the `nodeSelector` section of the `backend_options`.

21
pipeline/backend/docker/backend_options.go Normal file
View File

@ -0,0 +1,21 @@
package docker
import (
"github.com/mitchellh/mapstructure"
backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
)
// BackendOptions defines all the advanced options for the docker backend.
type BackendOptions struct {
User string `mapstructure:"user"`
}
func parseBackendOptions(step *backend.Step) (BackendOptions, error) {
var result BackendOptions
if step == nil || step.BackendOptions == nil {
return result, nil
}
err := mapstructure.Decode(step.BackendOptions[EngineName], &result)
return result, err
}

56
pipeline/backend/docker/backend_options_test.go Normal file
View File

@ -0,0 +1,56 @@
package docker
import (
"testing"
"github.com/stretchr/testify/assert"
backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
)
func Test_parseBackendOptions(t *testing.T) {
tests := []struct {
name string
step *backend.Step
want BackendOptions
wantErr bool
}{
{
name: "nil options",
step: &backend.Step{BackendOptions: nil},
want: BackendOptions{},
},
{
name: "empty options",
step: &backend.Step{BackendOptions: map[string]any{}},
want: BackendOptions{},
},
{
name: "with user option",
step: &backend.Step{BackendOptions: map[string]any{
"docker": map[string]any{
"user": "1000:1000",
},
}},
want: BackendOptions{User: "1000:1000"},
},
{
name: "invalid backend options",
step: &backend.Step{BackendOptions: map[string]any{"docker": "invalid"}},
want: BackendOptions{},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseBackendOptions(tt.step)
if tt.wantErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
assert.Equal(t, tt.want, got)
})
}
}

3
pipeline/backend/docker/convert.go
View File

@ -31,7 +31,7 @@ import (
const minVolumeComponents = 2
// returns a container configuration.
func (e *docker) toConfig(step *types.Step) *container.Config {
func (e *docker) toConfig(step *types.Step, options BackendOptions) *container.Config {
e.windowsPathPatch(step)
config := &container.Config{
@ -44,6 +44,7 @@ func (e *docker) toConfig(step *types.Step) *container.Config {
AttachStdout: true,
AttachStderr: true,
Volumes: toVol(step.Volumes),
User: options.User,
}
configEnv := make(map[string]string)
maps.Copy(configEnv, step.Environment)

10
pipeline/backend/docker/convert_test.go
View File

@ -131,7 +131,7 @@ func TestToContainerName(t *testing.T) {
func TestStepToConfig(t *testing.T) {
// StepTypeCommands
conf := testEngine.toConfig(testCmdStep)
conf := testEngine.toConfig(testCmdStep, BackendOptions{})
if assert.NotNil(t, conf) {
assert.EqualValues(t, []string{"/bin/sh", "-c", "echo $CI_SCRIPT | base64 -d | /bin/sh -e"}, conf.Entrypoint)
assert.Nil(t, conf.Cmd)
@ -139,7 +139,7 @@ func TestStepToConfig(t *testing.T) {
}
// StepTypePlugin
conf = testEngine.toConfig(testPluginStep)
conf = testEngine.toConfig(testPluginStep, BackendOptions{})
if assert.NotNil(t, conf) {
assert.Nil(t, conf.Cmd)
assert.EqualValues(t, testPluginStep.UUID, conf.Labels["wp_uuid"])
@ -174,7 +174,7 @@ func TestToConfigSmall(t *testing.T) {
Name: "test",
UUID: "09238932",
Commands: []string{"go test"},
})
}, BackendOptions{})
assert.NotNil(t, conf)
sort.Strings(conf.Env)
@ -233,7 +233,7 @@ func TestToConfigFull(t *testing.T) {
AuthConfig: backend.Auth{Username: "user", Password: "123456"},
NetworkMode: "bridge",
Ports: []backend.Port{{Number: 21}, {Number: 22}},
})
}, BackendOptions{})
assert.NotNil(t, conf)
sort.Strings(conf.Env)
@ -286,7 +286,7 @@ func TestToWindowsConfig(t *testing.T) {
AuthConfig: backend.Auth{Username: "user", Password: "123456"},
NetworkMode: "nat",
Ports: []backend.Port{{Number: 21}, {Number: 22}},
})
}, BackendOptions{})
assert.NotNil(t, conf)
sort.Strings(conf.Env)

12
pipeline/backend/docker/docker.go
View File

@ -46,6 +46,7 @@ type docker struct {
}
const (
EngineName = "docker"
networkDriverNAT = "nat"
networkDriverBridge = "bridge"
volumeDriver = "local"
@ -59,7 +60,7 @@ func New() backend.Backend {
}
func (e *docker) Name() string {
return "docker"
return EngineName
}
func (e *docker) IsAvailable(ctx context.Context) bool {
@ -170,9 +171,14 @@ func (e *docker) SetupWorkflow(ctx context.Context, conf *backend.Config, taskUU
}
func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID string) error {
options, err := parseBackendOptions(step)
if err != nil {
log.Error().Err(err).Msg("could not parse backend options")
}
log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name)
config := e.toConfig(step)
config := e.toConfig(step, options)
hostConfig := toHostConfig(step, &e.config)
containerName := toContainerName(step)
@ -204,7 +210,7 @@ func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID str
// add default volumes to the host configuration
hostConfig.Binds = utils.DeduplicateStrings(append(hostConfig.Binds, e.config.volumes...))
_, err := e.client.ContainerCreate(ctx, config, hostConfig, nil, nil, containerName)
_, err = e.client.ContainerCreate(ctx, config, hostConfig, nil, nil, containerName)
if client.IsErrNotFound(err) {
// automatically pull and try to re-create the image if the
// failure is caused because the image does not exist.

2
pipeline/backend/kubernetes/backend_options.go
View File

@ -86,7 +86,7 @@ const (
func parseBackendOptions(step *backend.Step) (BackendOptions, error) {
var result BackendOptions
if step.BackendOptions == nil {
if step == nil || step.BackendOptions == nil {
return result, nil
}
err := mapstructure.Decode(step.BackendOptions[EngineName], &result)

197
pipeline/backend/kubernetes/backend_options_test.go
View File

@ -9,97 +9,122 @@ import (
)
func Test_parseBackendOptions(t *testing.T) {
got, err := parseBackendOptions(&backend.Step{BackendOptions: nil})
assert.NoError(t, err)
assert.Equal(t, BackendOptions{}, got)
got, err = parseBackendOptions(&backend.Step{BackendOptions: map[string]any{}})
assert.NoError(t, err)
assert.Equal(t, BackendOptions{}, got)
got, err = parseBackendOptions(&backend.Step{
BackendOptions: map[string]any{
"kubernetes": map[string]any{
"nodeSelector": map[string]string{"storage": "ssd"},
"serviceAccountName": "wp-svc-acc",
"labels": map[string]string{"app": "test"},
"annotations": map[string]string{"apps.kubernetes.io/pod-index": "0"},
"tolerations": []map[string]any{
{"key": "net-port", "value": "100Mbit", "effect": TaintEffectNoSchedule},
},
"resources": map[string]any{
"requests": map[string]string{"memory": "128Mi", "cpu": "1000m"},
"limits": map[string]string{"memory": "256Mi", "cpu": "2"},
},
"securityContext": map[string]any{
"privileged": newBool(true),
"runAsNonRoot": newBool(true),
"runAsUser": newInt64(101),
"runAsGroup": newInt64(101),
"fsGroup": newInt64(101),
"seccompProfile": map[string]any{
"type": "Localhost",
"localhostProfile": "profiles/audit.json",
},
"apparmorProfile": map[string]any{
"type": "Localhost",
"localhostProfile": "k8s-apparmor-example-deny-write",
},
},
"secrets": []map[string]any{
{
"name": "aws",
"key": "access-key",
"target": map[string]any{
"env": "AWS_SECRET_ACCESS_KEY",
tests := []struct {
name string
step *backend.Step
want BackendOptions
wantErr bool
}{
{
name: "nil options",
step: &backend.Step{BackendOptions: nil},
want: BackendOptions{},
},
{
name: "empty options",
step: &backend.Step{BackendOptions: map[string]any{}},
want: BackendOptions{},
},
{
name: "full k8s options",
step: &backend.Step{
BackendOptions: map[string]any{
"kubernetes": map[string]any{
"nodeSelector": map[string]string{"storage": "ssd"},
"serviceAccountName": "wp-svc-acc",
"labels": map[string]string{"app": "test"},
"annotations": map[string]string{"apps.kubernetes.io/pod-index": "0"},
"tolerations": []map[string]any{
{"key": "net-port", "value": "100Mbit", "effect": TaintEffectNoSchedule},
},
},
{
"name": "reg-cred",
"key": ".dockerconfigjson",
"target": map[string]any{
"file": "~/.docker/config.json",
"resources": map[string]any{
"requests": map[string]string{"memory": "128Mi", "cpu": "1000m"},
"limits": map[string]string{"memory": "256Mi", "cpu": "2"},
},
"securityContext": map[string]any{
"privileged": newBool(true),
"runAsNonRoot": newBool(true),
"runAsUser": newInt64(101),
"runAsGroup": newInt64(101),
"fsGroup": newInt64(101),
"seccompProfile": map[string]any{
"type": "Localhost",
"localhostProfile": "profiles/audit.json",
},
"apparmorProfile": map[string]any{
"type": "Localhost",
"localhostProfile": "k8s-apparmor-example-deny-write",
},
},
"secrets": []map[string]any{
{
"name": "aws",
"key": "access-key",
"target": map[string]any{
"env": "AWS_SECRET_ACCESS_KEY",
},
},
{
"name": "reg-cred",
"key": ".dockerconfigjson",
"target": map[string]any{
"file": "~/.docker/config.json",
},
},
},
},
},
},
},
})
assert.NoError(t, err)
assert.Equal(t, BackendOptions{
NodeSelector: map[string]string{"storage": "ssd"},
ServiceAccountName: "wp-svc-acc",
Labels: map[string]string{"app": "test"},
Annotations: map[string]string{"apps.kubernetes.io/pod-index": "0"},
Tolerations: []Toleration{{Key: "net-port", Value: "100Mbit", Effect: TaintEffectNoSchedule}},
Resources: Resources{
Requests: map[string]string{"memory": "128Mi", "cpu": "1000m"},
Limits: map[string]string{"memory": "256Mi", "cpu": "2"},
},
SecurityContext: &SecurityContext{
Privileged: newBool(true),
RunAsNonRoot: newBool(true),
RunAsUser: newInt64(101),
RunAsGroup: newInt64(101),
FSGroup: newInt64(101),
SeccompProfile: &SecProfile{
Type: "Localhost",
LocalhostProfile: "profiles/audit.json",
},
ApparmorProfile: &SecProfile{
Type: "Localhost",
LocalhostProfile: "k8s-apparmor-example-deny-write",
want: BackendOptions{
NodeSelector: map[string]string{"storage": "ssd"},
ServiceAccountName: "wp-svc-acc",
Labels: map[string]string{"app": "test"},
Annotations: map[string]string{"apps.kubernetes.io/pod-index": "0"},
Tolerations: []Toleration{{Key: "net-port", Value: "100Mbit", Effect: TaintEffectNoSchedule}},
Resources: Resources{
Requests: map[string]string{"memory": "128Mi", "cpu": "1000m"},
Limits: map[string]string{"memory": "256Mi", "cpu": "2"},
},
SecurityContext: &SecurityContext{
Privileged: newBool(true),
RunAsNonRoot: newBool(true),
RunAsUser: newInt64(101),
RunAsGroup: newInt64(101),
FSGroup: newInt64(101),
SeccompProfile: &SecProfile{
Type: "Localhost",
LocalhostProfile: "profiles/audit.json",
},
ApparmorProfile: &SecProfile{
Type: "Localhost",
LocalhostProfile: "k8s-apparmor-example-deny-write",
},
},
Secrets: []SecretRef{
{
Name: "aws",
Key: "access-key",
Target: SecretTarget{Env: "AWS_SECRET_ACCESS_KEY"},
},
{
Name: "reg-cred",
Key: ".dockerconfigjson",
Target: SecretTarget{File: "~/.docker/config.json"},
},
},
},
},
Secrets: []SecretRef{
{
Name: "aws",
Key: "access-key",
Target: SecretTarget{Env: "AWS_SECRET_ACCESS_KEY"},
},
{
Name: "reg-cred",
Key: ".dockerconfigjson",
Target: SecretTarget{File: "~/.docker/config.json"},
},
},
}, got)
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseBackendOptions(tt.step)
if tt.wantErr {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.Equal(t, tt.want, got)
})
}
}