self-hosted/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-11-24 08:02:18 +02:00
Code Issues Releases Activity

Add backend selection for agent (#463)

Browse Source 
- add backend selection option
- by default it will auto-detect a backend
This commit is contained in:
Anbraten 2021-11-26 03:34:48 +01:00 committed by GitHub
parent 65e10d46b3
commit c1a8884d62
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 250 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
agent/runner.go
View File

@ -28,7 +28,7 @@ import (
"google.golang.org/grpc/metadata"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/multipart"
"github.com/woodpecker-ci/woodpecker/pipeline/rpc"
)

6
cli/exec/exec.go
View File

@ -15,8 +15,8 @@ import (
"github.com/woodpecker-ci/woodpecker/cli/common"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler"
@ -158,8 +158,8 @@ func execWithAxis(c *cli.Context, axis matrix.Axis) error {
compiler.WithSecret(secrets...),
compiler.WithEnviron(droneEnv),
).Compile(conf)
engine, err := docker.NewEnv()
if err != nil {
engine := docker.New()
if err = engine.Load(); err != nil {
return err
}

17
cmd/agent/agent.go
View File

@ -31,7 +31,7 @@ import (
"google.golang.org/grpc/metadata"
"github.com/woodpecker-ci/woodpecker/agent"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
"github.com/woodpecker-ci/woodpecker/pipeline/rpc"
)
@ -138,13 +138,22 @@ func loop(c *cli.Context) error {
return
}
// new docker engine
engine, err := docker.NewEnv()
// new engine
engine, err := backend.FindEngine(c.String("backend-engine"))
if err != nil {
log.Error().Err(err).Msg("cannot create docker client")
log.Error().Err(err).Msgf("cannot find backend engine '%s'", c.String("backend-engine"))
return
}
// load enginge (e.g. init api client)
err = engine.Load()
if err != nil {
log.Error().Err(err).Msg("cannot load backend engine")
return
}
log.Debug().Msgf("loaded %s backend engine", engine.Name())
r := agent.NewRunner(client, filter, hostname, counter, &engine)
if err := r.Run(ctx); err != nil {
log.Error().Err(err).Msg("pipeline done with error")

6
cmd/agent/flags.go
View File

@ -110,4 +110,10 @@ var flags = []cli.Flag{
Usage: "should the grpc server certificate be verified, only valid when WOODPECKER_GRPC_SECURE is true",
Value: true,
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND"},
Name: "backend-engine",
Usage: "backend engine to run pipelines on",
Value: "auto-detect",
},
}

55
pipeline/backend/backend.go
View File

@ -1,29 +1,44 @@
package backend
import (
"context"
"io"
"fmt"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
)
// Engine defines a container orchestration backend and is used
// to create and manage container resources.
type Engine interface {
// Setup the pipeline environment.
Setup(context.Context, *Config) error
var (
engines map[string]types.Engine
)
// Exec start the pipeline step.
Exec(context.Context, *Step) error
func init() {
engines = make(map[string]types.Engine)
// Kill the pipeline step.
Kill(context.Context, *Step) error
// TODO: disabled for now as kubernetes backend has not been implemented yet
// kubernetes
// engine = kubernetes.New("", "", "")
// engines[engine.Name()] = engine
// Wait for the pipeline step to complete and returns
// the completion results.
Wait(context.Context, *Step) (*State, error)
// Tail the pipeline step logs.
Tail(context.Context, *Step) (io.ReadCloser, error)
// Destroy the pipeline environment.
Destroy(context.Context, *Config) error
// docker
engine := docker.New()
engines[engine.Name()] = engine
}
func FindEngine(engineName string) (types.Engine, error) {
if engineName == "auto-detect" {
for _, engine := range engines {
if engine.IsAvivable() {
return engine, nil
}
}
return nil, fmt.Errorf("Can't detect an avivable backend engine")
}
engine, ok := engines[engineName]
if !ok {
return nil, fmt.Errorf("Backend engine '%s' not found", engineName)
}
return engine, nil
}

8
pipeline/backend/docker/convert.go
View File

@ -8,11 +8,11 @@ import (
"github.com/docker/docker/api/types/container"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
)
// returns a container configuration.
func toConfig(proc *backend.Step) *container.Config {
func toConfig(proc *types.Step) *container.Config {
config := &container.Config{
Image: proc.Image,
Labels: proc.Labels,
@ -36,7 +36,7 @@ func toConfig(proc *backend.Step) *container.Config {
}
// returns a container host configuration.
func toHostConfig(proc *backend.Step) *container.HostConfig {
func toHostConfig(proc *types.Step) *container.HostConfig {
config := &container.HostConfig{
Resources: container.Resources{
CPUQuota: proc.CPUQuota,
@ -146,7 +146,7 @@ func toDev(paths []string) []container.DeviceMapping {
// helper function that serializes the auth configuration as JSON
// base64 payload.
func encodeAuthToBase64(authConfig backend.Auth) (string, error) {
func encodeAuthToBase64(authConfig types.Auth) (string, error) {
buf, err := json.Marshal(authConfig)
if err != nil {
return "", err

28
pipeline/backend/docker/docker.go
View File

@ -14,28 +14,38 @@ import (
"github.com/moby/term"
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
)
type engine struct {
client client.APIClient
}
// New returns a new Docker Engine using the given client.
func New(cli client.APIClient) backend.Engine {
// New returns a new Docker Engine.
func New() backend.Engine {
return &engine{
client: cli,
client: nil,
}
}
// NewEnv returns a new Docker Engine using the client connection
// environment variables.
func NewEnv() (backend.Engine, error) {
func (e *engine) Name() string {
return "docker"
}
func (e *engine) IsAvivable() bool {
_, err := os.Stat("/.dockerenv")
return os.IsNotExist(err)
}
// Load new client for Docker Engine using environment variables.
func (e *engine) Load() error {
cli, err := client.NewClientWithOpts(client.FromEnv)
if err != nil {
return nil, err
return err
}
return New(cli), nil
e.client = cli
return nil
}
func (e *engine) Setup(_ context.Context, conf *backend.Config) error {

30
pipeline/backend/kubernetes/kubernetes.go
View File

@ -3,8 +3,9 @@ package kubernetes
import (
"context"
"io"
"os"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
)
type engine struct {
@ -14,7 +15,7 @@ type engine struct {
}
// New returns a new Kubernetes Engine.
func New(namespace, endpoint, token string) backend.Engine {
func New(namespace, endpoint, token string) types.Engine {
return &engine{
namespace: namespace,
endpoint: endpoint,
@ -22,40 +23,53 @@ func New(namespace, endpoint, token string) backend.Engine {
}
}
func (e *engine) Name() string {
return "kubernetes"
}
func (e *engine) IsAvivable() bool {
host := os.Getenv("KUBERNETES_SERVICE_HOST")
return len(host) > 0
}
func (e *engine) Load() error {
return nil
}
// Setup the pipeline environment.
func (e *engine) Setup(context.Context, *backend.Config) error {
func (e *engine) Setup(context.Context, *types.Config) error {
// POST /api/v1/namespaces
return nil
}
// Start the pipeline step.
func (e *engine) Exec(context.Context, *backend.Step) error {
func (e *engine) Exec(context.Context, *types.Step) error {
// POST /api/v1/namespaces/{namespace}/pods
return nil
}
// DEPRECATED
// Kill the pipeline step.
func (e *engine) Kill(context.Context, *backend.Step) error {
func (e *engine) Kill(context.Context, *types.Step) error {
return nil
}
// Wait for the pipeline step to complete and returns
// the completion results.
func (e *engine) Wait(context.Context, *backend.Step) (*backend.State, error) {
func (e *engine) Wait(context.Context, *types.Step) (*types.State, error) {
// GET /api/v1/watch/namespaces/{namespace}/pods
// GET /api/v1/watch/namespaces/{namespace}/pods/{name}
return nil, nil
}
// Tail the pipeline step logs.
func (e *engine) Tail(context.Context, *backend.Step) (io.ReadCloser, error) {
func (e *engine) Tail(context.Context, *types.Step) (io.ReadCloser, error) {
// GET /api/v1/namespaces/{namespace}/pods/{name}/log
return nil, nil
}
// Destroy the pipeline environment.
func (e *engine) Destroy(context.Context, *backend.Config) error {
func (e *engine) Destroy(context.Context, *types.Config) error {
// DELETE /api/v1/namespaces/{name}
return nil
}

116
pipeline/backend/types.go
View File

@ -1,116 +0,0 @@
package backend
type (
// Config defines the runtime configuration of a pipeline.
Config struct {
Stages []*Stage `json:"pipeline"` // pipeline stages
Networks []*Network `json:"networks"` // network definitions
Volumes []*Volume `json:"volumes"` // volume definitions
Secrets []*Secret `json:"secrets"` // secret definitions
}
// Stage denotes a collection of one or more steps.
Stage struct {
Name string `json:"name,omitempty"`
Alias string `json:"alias,omitempty"`
Steps []*Step `json:"steps,omitempty"`
}
// Step defines a container process.
Step struct {
Name string `json:"name"`
Alias string `json:"alias,omitempty"`
Image string `json:"image,omitempty"`
Pull bool `json:"pull,omitempty"`
Detached bool `json:"detach,omitempty"`
Privileged bool `json:"privileged,omitempty"`
WorkingDir string `json:"working_dir,omitempty"`
Environment map[string]string `json:"environment,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
Entrypoint []string `json:"entrypoint,omitempty"`
Command []string `json:"command,omitempty"`
ExtraHosts []string `json:"extra_hosts,omitempty"`
Volumes []string `json:"volumes,omitempty"`
Tmpfs []string `json:"tmpfs,omitempty"`
Devices []string `json:"devices,omitempty"`
Networks []Conn `json:"networks,omitempty"`
DNS []string `json:"dns,omitempty"`
DNSSearch []string `json:"dns_search,omitempty"`
MemSwapLimit int64 `json:"memswap_limit,omitempty"`
MemLimit int64 `json:"mem_limit,omitempty"`
ShmSize int64 `json:"shm_size,omitempty"`
CPUQuota int64 `json:"cpu_quota,omitempty"`
CPUShares int64 `json:"cpu_shares,omitempty"`
CPUSet string `json:"cpu_set,omitempty"`
OnFailure bool `json:"on_failure,omitempty"`
OnSuccess bool `json:"on_success,omitempty"`
AuthConfig Auth `json:"auth_config,omitempty"`
NetworkMode string `json:"network_mode,omitempty"`
IpcMode string `json:"ipc_mode,omitempty"`
Sysctls map[string]string `json:"sysctls,omitempty"`
}
// Auth defines registry authentication credentials.
Auth struct {
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
Email string `json:"email,omitempty"`
}
// Conn defines a container network connection.
Conn struct {
Name string `json:"name"`
Aliases []string `json:"aliases"`
}
// Network defines a container network.
Network struct {
Name string `json:"name,omitempty"`
Driver string `json:"driver,omitempty"`
DriverOpts map[string]string `json:"driver_opts,omitempty"`
}
// Volume defines a container volume.
Volume struct {
Name string `json:"name,omitempty"`
Driver string `json:"driver,omitempty"`
DriverOpts map[string]string `json:"driver_opts,omitempty"`
}
// Secret defines a runtime secret
Secret struct {
Name string `json:"name,omitempty"`
Value string `json:"value,omitempty"`
Mount string `json:"mount,omitempty"`
Mask bool `json:"mask,omitempty"`
}
// State defines a container state.
State struct {
// Container exit code
ExitCode int `json:"exit_code"`
// Container exited, true or false
Exited bool `json:"exited"`
// Container is oom killed, true or false
OOMKilled bool `json:"oom_killed"`
}
// // State defines the pipeline and process state.
// State struct {
// Pipeline struct {
// // Current pipeline step
// Step *Step `json:"step"`
// // Current pipeline error state
// Error error `json:"error"`
// }
//
// Process struct {
// // Container exit code
// ExitCode int `json:"exit_code"`
// // Container exited, true or false
// Exited bool `json:"exited"`
// // Container is oom killed, true or false
// OOMKilled bool `json:"oom_killed"`
// }
// }
)

8
pipeline/backend/types/auth.go Normal file
View File

@ -0,0 +1,8 @@
package types
// Auth defines registry authentication credentials.
type Auth struct {
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
Email string `json:"email,omitempty"`
}

9
pipeline/backend/types/config.go Normal file
View File

@ -0,0 +1,9 @@
package types
// Config defines the runtime configuration of a pipeline.
type Config struct {
Stages []*Stage `json:"pipeline"` // pipeline stages
Networks []*Network `json:"networks"` // network definitions
Volumes []*Volume `json:"volumes"` // volume definitions
Secrets []*Secret `json:"secrets"` // secret definitions
}

7
pipeline/backend/types/conn.go Normal file
View File

@ -0,0 +1,7 @@
package types
// Conn defines a container network connection.
type Conn struct {
Name string `json:"name"`
Aliases []string `json:"aliases"`
}

35
pipeline/backend/types/engine.go Normal file
View File

@ -0,0 +1,35 @@
package types
import (
"context"
"io"
)
// Engine defines a container orchestration backend and is used
// to create and manage container resources.
type Engine interface {
Name() string
IsAvivable() bool
Load() error
// Setup the pipeline environment.
Setup(context.Context, *Config) error
// Exec start the pipeline step.
Exec(context.Context, *Step) error
// Kill the pipeline step.
Kill(context.Context, *Step) error
// Wait for the pipeline step to complete and returns
// the completion results.
Wait(context.Context, *Step) (*State, error)
// Tail the pipeline step logs.
Tail(context.Context, *Step) (io.ReadCloser, error)
// Destroy the pipeline environment.
Destroy(context.Context, *Config) error
}

8
pipeline/backend/types/network.go Normal file
View File

@ -0,0 +1,8 @@
package types
// Network defines a container network.
type Network struct {
Name string `json:"name,omitempty"`
Driver string `json:"driver,omitempty"`
DriverOpts map[string]string `json:"driver_opts,omitempty"`
}

9
pipeline/backend/types/secret.go Normal file
View File

@ -0,0 +1,9 @@
package types
// Secret defines a runtime secret
type Secret struct {
Name string `json:"name,omitempty"`
Value string `json:"value,omitempty"`
Mount string `json:"mount,omitempty"`
Mask bool `json:"mask,omitempty"`
}

8
pipeline/backend/types/stage.go Normal file
View File

@ -0,0 +1,8 @@
package types
// Stage denotes a collection of one or more steps.
type Stage struct {
Name string `json:"name,omitempty"`
Alias string `json:"alias,omitempty"`
Steps []*Step `json:"steps,omitempty"`
}

11
pipeline/backend/types/state.go Normal file
View File

@ -0,0 +1,11 @@
package types
// State defines a container state.
type State struct {
// Container exit code
ExitCode int `json:"exit_code"`
// Container exited, true or false
Exited bool `json:"exited"`
// Container is oom killed, true or false
OOMKilled bool `json:"oom_killed"`
}

35
pipeline/backend/types/step.go Normal file
View File

@ -0,0 +1,35 @@
package types
// Step defines a container process.
type Step struct {
Name string `json:"name"`
Alias string `json:"alias,omitempty"`
Image string `json:"image,omitempty"`
Pull bool `json:"pull,omitempty"`
Detached bool `json:"detach,omitempty"`
Privileged bool `json:"privileged,omitempty"`
WorkingDir string `json:"working_dir,omitempty"`
Environment map[string]string `json:"environment,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
Entrypoint []string `json:"entrypoint,omitempty"`
Command []string `json:"command,omitempty"`
ExtraHosts []string `json:"extra_hosts,omitempty"`
Volumes []string `json:"volumes,omitempty"`
Tmpfs []string `json:"tmpfs,omitempty"`
Devices []string `json:"devices,omitempty"`
Networks []Conn `json:"networks,omitempty"`
DNS []string `json:"dns,omitempty"`
DNSSearch []string `json:"dns_search,omitempty"`
MemSwapLimit int64 `json:"memswap_limit,omitempty"`
MemLimit int64 `json:"mem_limit,omitempty"`
ShmSize int64 `json:"shm_size,omitempty"`
CPUQuota int64 `json:"cpu_quota,omitempty"`
CPUShares int64 `json:"cpu_shares,omitempty"`
CPUSet string `json:"cpu_set,omitempty"`
OnFailure bool `json:"on_failure,omitempty"`
OnSuccess bool `json:"on_success,omitempty"`
AuthConfig Auth `json:"auth_config,omitempty"`
NetworkMode string `json:"network_mode,omitempty"`
IpcMode string `json:"ipc_mode,omitempty"`
Sysctls map[string]string `json:"sysctls,omitempty"`
}

8
pipeline/backend/types/volume.go Normal file
View File

@ -0,0 +1,8 @@
package types
// Volume defines a container volume.
type Volume struct {
Name string `json:"name,omitempty"`
Driver string `json:"driver,omitempty"`
DriverOpts map[string]string `json:"driver_opts,omitempty"`
}

2
pipeline/frontend/yaml/compiler/compiler.go
View File

@ -3,7 +3,7 @@ package compiler
import (
"fmt"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
)

2
pipeline/frontend/yaml/compiler/convert.go
View File

@ -7,7 +7,7 @@ import (
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
)

2
pipeline/logger.go
View File

@ -1,7 +1,7 @@
package pipeline
import (
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/multipart"
)

2
pipeline/option.go
View File

@ -3,7 +3,7 @@ package pipeline
import (
"context"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
)
// Option configures a runtime option.

2
pipeline/parse.go
View File

@ -6,7 +6,7 @@ import (
"os"
"strings"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
)
// Parse parses the pipeline config from an io.Reader.

2
pipeline/pipeline.go
View File

@ -8,7 +8,7 @@ import (
"github.com/rs/zerolog/log"
"golang.org/x/sync/errgroup"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/multipart"
)

2
pipeline/rpc/client_grpc.go
View File

@ -10,7 +10,7 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/rpc/proto"
)

2
pipeline/rpc/peer.go
View File

@ -3,7 +3,7 @@ package rpc
import (
"context"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
)
type (

2
server/shared/procBuilder.go
View File

@ -24,7 +24,7 @@ import (
"github.com/drone/envsubst"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler"