From 80c72b590c5e9942782a4580e7618fa0005d0be8 Mon Sep 17 00:00:00 2001 From: Anthony Wang Date: Thu, 10 Mar 2022 15:07:02 -0600 Subject: [PATCH] Add support to run pipelines using a local backend (#709) This adds support for #559. I tested using [this .woodpecker.yml](https://git.exozy.me/Ta180m/Hello-world/src/branch/main/.woodpecker.yml) on my self-hosted [Woodpecker instance](https://ci.exozy.me/Ta180m/Hello-world). I was also able to get this to build [Hugo websites](https://ci.exozy.me/Ta180m/howtuwu/build/1). It's currently very simplistic but works! close #559 --- docs/docs/20-usage/20-pipeline-syntax.md | 8 +- .../docs/30-administration/15-agent-config.md | 2 +- pipeline/backend/backend.go | 2 + pipeline/backend/local/local.go | 111 ++++++++++++++++++ 4 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 pipeline/backend/local/local.go diff --git a/docs/docs/20-usage/20-pipeline-syntax.md b/docs/docs/20-usage/20-pipeline-syntax.md index 5bdc18b3b..6151dc9eb 100644 --- a/docs/docs/20-usage/20-pipeline-syntax.md +++ b/docs/docs/20-usage/20-pipeline-syntax.md @@ -164,7 +164,9 @@ pipeline: ### `image` -Woodpecker uses Docker images for the build environment, for plugins and for service containers. The image field is exposed in the container blocks in the Yaml: +With the `docker` backend, Woodpecker uses Docker images for the build environment, for plugins and for service containers. The image field is exposed in the container blocks in the Yaml: + +When using the `local` backend, the `image` entry is used to specify the shell, such as Bash or Fish, that is used to run the commands. ```diff pipeline: @@ -406,7 +408,9 @@ For more details check the [matrix build docs](/docs/usage/matrix-builds/). ### `clone` -Woodpecker automatically configures a default clone step if not explicitly defined. You can manually configure the clone step in your pipeline for customization: +Woodpecker automatically configures a default clone step if not explicitly defined. When using the `local` backend, the [plugin-git](https://github.com/woodpecker-ci/plugin-git) binary must be on your `$PATH` for the default clone step to work. If not, you can still write a manual clone step. + +You can manually configure the clone step in your pipeline for customization: ```diff +clone: diff --git a/docs/docs/30-administration/15-agent-config.md b/docs/docs/30-administration/15-agent-config.md index f5fda4926..f3c391491 100644 --- a/docs/docs/30-administration/15-agent-config.md +++ b/docs/docs/30-administration/15-agent-config.md @@ -153,4 +153,4 @@ Configures if the gRPC server certificate should be verified, only valid when `W ### `WOODPECKER_BACKEND` > Default: `auto-detect` -Configures the backend engine to run pipelines on. Possible values are `auto-detect` or `docker`. +Configures the backend engine to run pipelines on. Possible values are `auto-detect`, `docker`, or `local`. diff --git a/pipeline/backend/backend.go b/pipeline/backend/backend.go index 4d4ea33b3..26fd2983c 100644 --- a/pipeline/backend/backend.go +++ b/pipeline/backend/backend.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/woodpecker-ci/woodpecker/pipeline/backend/docker" + "github.com/woodpecker-ci/woodpecker/pipeline/backend/local" "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" ) @@ -12,6 +13,7 @@ var engines map[string]types.Engine func init() { loadedEngines := []types.Engine{ docker.New(), + local.New(), // kubernetes.New(), // TODO: disabled for now as kubernetes backend has not been implemented yet } diff --git a/pipeline/backend/local/local.go b/pipeline/backend/local/local.go new file mode 100644 index 000000000..8f07d1f73 --- /dev/null +++ b/pipeline/backend/local/local.go @@ -0,0 +1,111 @@ +package local + +import ( + "context" + "encoding/base64" + "io" + "os" + "os/exec" + "strings" + + "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" + "github.com/woodpecker-ci/woodpecker/server" +) + +type local struct { + cmd *exec.Cmd + output io.ReadCloser +} + +// make sure local implements Engine +var _ types.Engine = &local{} + +// New returns a new local Engine. +func New() types.Engine { + return &local{} +} + +func (e *local) Name() string { + return "local" +} + +func (e *local) IsAvailable() bool { + return true +} + +func (e *local) Load() error { + return nil +} + +// Setup the pipeline environment. +func (e *local) Setup(ctx context.Context, proc *types.Config) error { + return nil +} + +// Exec the pipeline step. +func (e *local) Exec(ctx context.Context, proc *types.Step) error { + // Get environment variables + Command := []string{} + for a, b := range proc.Environment { + if a != "HOME" && a != "SHELL" { // Don't override $HOME and $SHELL + Command = append(Command, a+"="+b) + } + } + + // Get default clone image + defaultCloneImage := "docker.io/woodpeckerci/plugin-git:latest" + if len(server.Config.Pipeline.DefaultCloneImage) > 0 { + defaultCloneImage = server.Config.Pipeline.DefaultCloneImage + } + + if proc.Image == defaultCloneImage { + // Default clone step + Command = append(Command, "CI_WORKSPACE=/tmp/woodpecker/"+proc.Environment["CI_REPO"]) + Command = append(Command, "plugin-git") + } else { + // Use "image name" as run command + Command = append(Command, proc.Image[18:len(proc.Image)-7]) + Command = append(Command, "-c") + + // Decode script and delete initial lines + // Deleting the initial lines removes netrc support but adds compatibility for more shells like fish + Script, _ := base64.RawStdEncoding.DecodeString(proc.Environment["CI_SCRIPT"]) + Command = append(Command, string(Script)[strings.Index(string(Script), "\n\n")+2:]) + } + + // Prepare command + e.cmd = exec.CommandContext(ctx, "/bin/env", Command...) + + // Prepare working directory + if proc.Image == defaultCloneImage { + e.cmd.Dir = "/tmp/woodpecker/" + proc.Environment["CI_REPO_OWNER"] + } else { + e.cmd.Dir = "/tmp/woodpecker/" + proc.Environment["CI_REPO"] + } + _ = os.MkdirAll(e.cmd.Dir, 0o700) + + // Get output and redirect Stderr to Stdout + e.output, _ = e.cmd.StdoutPipe() + e.cmd.Stderr = e.cmd.Stdout + + return e.cmd.Start() +} + +// Wait for the pipeline step to complete and returns +// the completion results. +func (e *local) Wait(context.Context, *types.Step) (*types.State, error) { + return &types.State{ + Exited: true, + }, e.cmd.Wait() +} + +// Tail the pipeline step logs. +func (e *local) Tail(context.Context, *types.Step) (io.ReadCloser, error) { + return e.output, nil +} + +// Destroy the pipeline environment. +func (e *local) Destroy(context.Context, *types.Config) error { + os.RemoveAll(e.cmd.Dir) + return nil +}