Make agent usable for external backends (#3270)

cli/exec/exec.go

@@ -213,7 +213,11 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error
 	}
 
 	backendCtx := context.WithValue(c.Context, backendTypes.CliContext, c)
-	backend.Init()
+	backend.Init([]backendTypes.Backend{
+		docker.New(),
+		local.New(),
+		kubernetes.New(),
+	})
 
 	backendEngine, err := backend.FindBackend(backendCtx, c.String("backend-engine"))
 	if err != nil {

cmd/agent/core/agent.go

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package core
 
 import (
 	"context"
@@ -42,14 +42,12 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
-	"go.woodpecker-ci.org/woodpecker/v2/shared/addon"
-	addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types"
 	"go.woodpecker-ci.org/woodpecker/v2/shared/logger"
 	"go.woodpecker-ci.org/woodpecker/v2/shared/utils"
 	"go.woodpecker-ci.org/woodpecker/v2/version"
 )
 
-func run(c *cli.Context) error {
+func run(c *cli.Context, backendEngines []types.Backend) error {
 	agentConfigPath := c.String("agent-config")
 	hostname := c.String("hostname")
 	if len(hostname) == 0 {
@@ -155,11 +153,13 @@ func run(c *cli.Context) error {
 
 	// new engine
 	backendCtx := context.WithValue(ctx, types.CliContext, c)
-	backendEngine, err := getBackendEngine(backendCtx, c.String("backend-engine"), c.StringSlice("addons"))
+	backend.Init(backendEngines)
+	backendName := c.String("backend-engine")
+	backendEngine, err := backend.FindBackend(backendCtx, backendName)
 	if err != nil {
+		log.Error().Err(err).Msgf("cannot find backend engine '%s'", backendName)
 		return err
 	}
-
 	if !backendEngine.IsAvailable(backendCtx) {
 		log.Error().Str("engine", backendEngine.Name()).Msg("selected backend engine is unavailable")
 		return fmt.Errorf("selected backend engine %s is unavailable", backendEngine.Name())
@@ -249,44 +249,27 @@ func run(c *cli.Context) error {
 	return nil
 }
 
-func getBackendEngine(backendCtx context.Context, backendName string, addons []string) (types.Backend, error) {
-	addonBackend, err := addon.Load[types.Backend](addons, addonTypes.TypeBackend)
-	if err != nil {
-		log.Error().Err(err).Msg("cannot load backend addon")
-		return nil, err
-	}
-	if addonBackend != nil {
-		return addonBackend.Value, nil
-	}
+func runWithRetry(backendEngines []types.Backend) func(context *cli.Context) error {
+	return func(context *cli.Context) error {
+		if err := logger.SetupGlobalLogger(context, true); err != nil {
+			return err
+		}
 
-	backend.Init()
-	engine, err := backend.FindBackend(backendCtx, backendName)
-	if err != nil {
-		log.Error().Err(err).Msgf("cannot find backend engine '%s'", backendName)
-		return nil, err
-	}
-	return engine, nil
-}
+		initHealth()
 
-func runWithRetry(context *cli.Context) error {
-	if err := logger.SetupGlobalLogger(context, true); err != nil {
+		retryCount := context.Int("connect-retry-count")
+		retryDelay := context.Duration("connect-retry-delay")
+		var err error
+		for i := 0; i < retryCount; i++ {
+			if err = run(context, backendEngines); status.Code(err) == codes.Unavailable {
+				log.Warn().Err(err).Msg(fmt.Sprintf("cannot connect to server, retrying in %v", retryDelay))
+				time.Sleep(retryDelay)
+			} else {
+				break
+			}
+		}
 		return err
 	}
-
-	initHealth()
-
-	retryCount := context.Int("connect-retry-count")
-	retryDelay := context.Duration("connect-retry-delay")
-	var err error
-	for i := 0; i < retryCount; i++ {
-		if err = run(context); status.Code(err) == codes.Unavailable {
-			log.Warn().Err(err).Msg(fmt.Sprintf("cannot connect to server, retrying in %v", retryDelay))
-			time.Sleep(retryDelay)
-		} else {
-			break
-		}
-	}
-	return err
 }
 
 func stringSliceAddToMap(sl []string, m map[string]string) error {

cmd/agent/core/agent_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package core
 
 import (
 	"testing"

cmd/agent/core/config.go

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package core
 
 import (
 	"bytes"

cmd/agent/core/config_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package core
 
 import (
 	"os"

cmd/agent/core/flags.go

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package core
 
 import (
 	"os"
@@ -97,9 +97,4 @@ var flags = []cli.Flag{
 		Usage:   "backend to run pipelines on",
 		Value:   "auto-detect",
 	},
-	&cli.StringSliceFlag{
-		EnvVars: []string{"WOODPECKER_ADDONS"},
-		Name:    "addons",
-		Usage:   "list of addon files",
-	},
 }

cmd/agent/core/health.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package core
 
 import (
 	"encoding/json"

cmd/agent/core/health_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package core
 
 import (
 	"testing"

cmd/agent/core/run.go

@@ -0,0 +1,53 @@
+// Copyright 2024 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package core
+
+import (
+	"os"
+
+	// Load config from .env
+	_ "github.com/joho/godotenv/autoload"
+	"github.com/rs/zerolog/log"
+	"github.com/urfave/cli/v2"
+
+	backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
+	"go.woodpecker-ci.org/woodpecker/v2/shared/logger"
+	"go.woodpecker-ci.org/woodpecker/v2/shared/utils"
+	"go.woodpecker-ci.org/woodpecker/v2/version"
+)
+
+func RunAgent(backends []backend.Backend) {
+	app := cli.NewApp()
+	app.Name = "woodpecker-agent"
+	app.Version = version.String()
+	app.Usage = "woodpecker agent"
+	app.Action = runWithRetry(backends)
+	app.Commands = []*cli.Command{
+		{
+			Name:   "ping",
+			Usage:  "ping the agent",
+			Action: pinger,
+		},
+	}
+	agentFlags := utils.MergeSlices(flags, logger.GlobalLoggerFlags)
+	for _, b := range backends {
+		agentFlags = utils.MergeSlices(agentFlags, b.Flags())
+	}
+	app.Flags = agentFlags
+
+	if err := app.Run(os.Args); err != nil {
+		log.Fatal().Err(err).Msg("error running agent") //nolint:forbidigo
+	}
+}

cmd/agent/main.go

@@ -15,36 +15,17 @@
 package main
 
 import (
-	"os"
-
-	_ "github.com/joho/godotenv/autoload"
-	"github.com/rs/zerolog/log"
-	"github.com/urfave/cli/v2"
-
+	"go.woodpecker-ci.org/woodpecker/v2/cmd/agent/core"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/docker"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/kubernetes"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/local"
-	"go.woodpecker-ci.org/woodpecker/v2/shared/logger"
-	"go.woodpecker-ci.org/woodpecker/v2/shared/utils"
-	"go.woodpecker-ci.org/woodpecker/v2/version"
+	backendTypes "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
 )
 
 func main() {
-	app := cli.NewApp()
-	app.Name = "woodpecker-agent"
-	app.Version = version.String()
-	app.Usage = "woodpecker agent"
-	app.Action = runWithRetry
-	app.Commands = []*cli.Command{
-		{
-			Name:   "ping",
-			Usage:  "ping the agent",
-			Action: pinger,
-		},
-	}
-	app.Flags = utils.MergeSlices(flags, logger.GlobalLoggerFlags, docker.Flags, kubernetes.Flags, local.Flags)
-
-	if err := app.Run(os.Args); err != nil {
-		log.Fatal().Err(err).Msg("error running agent") //nolint:forbidigo
-	}
+	core.RunAgent([]backendTypes.Backend{
+		docker.New(),
+		local.New(),
+		kubernetes.New(),
+	})
 }

docs/docs/30-administration/15-agent-config.md

@@ -168,12 +168,6 @@ Configures if the gRPC server certificate should be verified, only valid when `W
 
 Configures the backend engine to run pipelines on. Possible values are `auto-detect`, `docker`, `local` or `kubernetes`.
 
-### `WOODPECKER_ADDONS`
-
-> Default: empty
-
-List of addon files. See [addons](./75-addons/00-overview.md).
-
 ### `WOODPECKER_BACKEND_DOCKER_*`
 
 See [Docker backend configuration](./22-backends/10-docker.md#configuration)

docs/docs/30-administration/75-addons/00-overview.md

@@ -13,7 +13,6 @@ To adapt Woodpecker to your needs beyond the [configuration](../10-server-config
 Addons can be used for:
 
 - Forges
-- Agent backends
 - Config services
 - Secret services
 - Environment services

docs/docs/30-administration/75-addons/20-creating-addons.md

@@ -19,14 +19,13 @@ Directly import Woodpecker's Go package (`go.woodpecker-ci.org/woodpecker/woodpe
 
 ### Return types
 
-| Addon type           | Return type                                                                      |
-| -------------------- | -------------------------------------------------------------------------------- |
-| `Forge`              | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge`             |
-| `Backend`            | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/pipeline/backend/types".Backend` |
-| `ConfigService`      | `"go.woodpecker-ci.org/woodpecker/v2/server/plugins/config".Extension`           |
-| `SecretService`      | `"go.woodpecker-ci.org/woodpecker/v2/server/model".SecretService`                |
-| `EnvironmentService` | `"go.woodpecker-ci.org/woodpecker/v2/server/model".EnvironmentService`           |
-| `RegistryService`    | `"go.woodpecker-ci.org/woodpecker/v2/server/model".RegistryService`              |
+| Addon type           | Return type                                                            |
+| -------------------- | ---------------------------------------------------------------------- |
+| `Forge`              | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge`   |
+| `ConfigService`      | `"go.woodpecker-ci.org/woodpecker/v2/server/plugins/config".Extension` |
+| `SecretService`      | `"go.woodpecker-ci.org/woodpecker/v2/server/model".SecretService`      |
+| `EnvironmentService` | `"go.woodpecker-ci.org/woodpecker/v2/server/model".EnvironmentService` |
+| `RegistryService`    | `"go.woodpecker-ci.org/woodpecker/v2/server/model".RegistryService`    |
 
 ### Using configurations
 

docs/docs/92-development/10-custom-agent-backends.md

@@ -0,0 +1,23 @@
+# Custom agent backends
+
+If none of our backends fits your usecases, you can write your own.
+
+Therefore, implement the interface `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/pipeline/backend/types".Backend` and
+build a custom agent using your backend with this `main.go`:
+
+```go
+package main
+
+import (
+  "go.woodpecker-ci.org/woodpecker/v2/cmd/agent/core"
+  backendTypes "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
+)
+
+func main() {
+  core.RunAgent([]backendTypes.Backend{
+    yourBackend,
+  })
+}
+```
+
+It is also possible to use multiple backends, you can select with [`WOODPECKER_BACKEND`](../30-administration/15-agent-config.md#woodpecker_backend) between them.

pipeline/backend/backend.go

@@ -18,9 +18,6 @@ import (
 	"context"
 	"fmt"
 
-	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/docker"
-	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/kubernetes"
-	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/local"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
 )
 
@@ -29,13 +26,7 @@ var (
 	backends       []types.Backend
 )
 
-func Init() {
-	backends = []types.Backend{
-		docker.New(),
-		local.New(),
-		kubernetes.New(),
-	}
-
+func Init(backends []types.Backend) {
 	backendsByName = make(map[string]types.Backend)
 	for _, engine := range backends {
 		backendsByName[engine.Name()] = engine

pipeline/backend/docker/docker.go

@@ -93,6 +93,10 @@ func httpClientOfOpts(dockerCertPath string, verifyTLS bool) *http.Client {
 	}
 }
 
+func (e *docker) Flags() []cli.Flag {
+	return Flags
+}
+
 // Load new client for Docker Backend using environment variables.
 func (e *docker) Load(ctx context.Context) (*backend.BackendInfo, error) {
 	c, ok := ctx.Value(backend.CliContext).(*cli.Context)

pipeline/backend/kubernetes/kubernetes.go

@@ -129,6 +129,10 @@ func (e *kube) IsAvailable(context.Context) bool {
 	return len(host) > 0
 }
 
+func (e *kube) Flags() []cli.Flag {
+	return Flags
+}
+
 func (e *kube) Load(ctx context.Context) (*types.BackendInfo, error) {
 	config, err := configFromCliContext(ctx)
 	if err != nil {

pipeline/backend/local/local.go

@@ -66,6 +66,10 @@ func (e *local) IsAvailable(context.Context) bool {
 	return true
 }
 
+func (e *local) Flags() []cli.Flag {
+	return Flags
+}
+
 func (e *local) Load(ctx context.Context) (*types.BackendInfo, error) {
 	c, ok := ctx.Value(types.CliContext).(*cli.Context)
 	if ok {

pipeline/backend/types/backend.go

@@ -17,6 +17,8 @@ package types
 import (
 	"context"
 	"io"
+
+	"github.com/urfave/cli/v2"
 )
 
 // Backend defines a container orchestration backend and is used
@@ -28,6 +30,9 @@ type Backend interface {
 	// IsAvailable check if the backend is available.
 	IsAvailable(ctx context.Context) bool
 
+	// Flags return the configuration flags of the backend.
+	Flags() []cli.Flag
+
 	// Load loads the backend engine.
 	Load(ctx context.Context) (*BackendInfo, error)
 

shared/addon/types/types.go

@@ -4,7 +4,6 @@ type Type string
 
 const (
 	TypeForge              Type = "forge"
-	TypeBackend            Type = "backend"
 	TypeConfigService      Type = "config_service"
 	TypeSecretService      Type = "secret_service"
 	TypeEnvironmentService Type = "environment_service"