From d253f8cc3018ddfd095696201c4a0272e80b5f8d Mon Sep 17 00:00:00 2001 From: 6543 Date: Mon, 7 Aug 2023 21:13:26 +0200 Subject: [PATCH] Make sure we dont have hidden options for backend and pipeline compiler (#2123) move options based on **os.Getenv** into flags --------- *Sponsored by Kithara Software GmbH* --- cli/exec/exec.go | 12 ++- cli/exec/flags.go | 102 ++++-------------- cmd/agent/flags.go | 101 +---------------- cmd/agent/main.go | 8 +- cmd/server/flags.go | 18 ++++ cmd/server/server.go | 5 + pipeline/backend/docker/docker.go | 51 +++++++-- pipeline/backend/docker/flags.go | 59 ++++++++++ pipeline/backend/kubernetes/flags.go | 72 +++++++++++++ pipeline/backend/kubernetes/kubernetes.go | 14 +++ pipeline/backend/kubernetes/pod.go | 14 +++ pipeline/backend/kubernetes/service.go | 14 +++ pipeline/backend/kubernetes/service_test.go | 14 +++ pipeline/backend/kubernetes/utils.go | 22 +++- pipeline/backend/kubernetes/volume.go | 14 +++ pipeline/backend/kubernetes/volume_test.go | 14 +++ pipeline/backend/local/flags.go | 21 ++++ pipeline/backend/ssh/flags.go | 47 ++++++++ pipeline/backend/ssh/ssh.go | 14 +++ pipeline/frontend/yaml/compiler/option.go | 71 ++++-------- .../frontend/yaml/compiler/option_test.go | 47 ++------ pipeline/stepBuilder.go | 23 ++-- server/config.go | 5 + server/pipeline/items.go | 6 ++ shared/utils/paginate.go | 14 +++ shared/utils/paginate_test.go | 14 +++ shared/utils/slices.go | 33 ++++++ shared/utils/slices_test.go | 29 +++++ 28 files changed, 562 insertions(+), 296 deletions(-) create mode 100644 pipeline/backend/docker/flags.go create mode 100644 pipeline/backend/kubernetes/flags.go create mode 100644 pipeline/backend/local/flags.go create mode 100644 pipeline/backend/ssh/flags.go create mode 100644 shared/utils/slices.go create mode 100644 shared/utils/slices_test.go diff --git a/cli/exec/exec.go b/cli/exec/exec.go index 63523d02e..dacad2745 100644 --- a/cli/exec/exec.go +++ b/cli/exec/exec.go @@ -30,6 +30,10 @@ import ( "github.com/woodpecker-ci/woodpecker/cli/common" "github.com/woodpecker-ci/woodpecker/pipeline" "github.com/woodpecker-ci/woodpecker/pipeline/backend" + "github.com/woodpecker-ci/woodpecker/pipeline/backend/docker" + "github.com/woodpecker-ci/woodpecker/pipeline/backend/kubernetes" + "github.com/woodpecker-ci/woodpecker/pipeline/backend/local" + "github.com/woodpecker-ci/woodpecker/pipeline/backend/ssh" backendTypes "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler" @@ -45,7 +49,7 @@ var Command = &cli.Command{ Usage: "execute a local pipeline", ArgsUsage: "[path/to/.woodpecker.yaml]", Action: run, - Flags: append(common.GlobalFlags, flags...), + Flags: utils.MergeSlices(common.GlobalFlags, flags, docker.Flags, ssh.Flags, kubernetes.Flags, local.Flags), } func run(c *cli.Context) error { @@ -184,7 +188,11 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error compiler.WithPrefix( c.String("prefix"), ), - compiler.WithProxy(), + compiler.WithProxy(compiler.ProxyOptions{ + NoProxy: c.String("backend-no-proxy"), + HTTPProxy: c.String("backend-http-proxy"), + HTTPSProxy: c.String("backend-https-proxy"), + }), compiler.WithLocal( c.Bool("local"), ), diff --git a/cli/exec/flags.go b/cli/exec/flags.go index b4692f46a..ba29546ea 100644 --- a/cli/exec/flags.go +++ b/cli/exec/flags.go @@ -64,6 +64,25 @@ var flags = []cli.Flag{ Value: "auto-detect", }, + // + // backend options for pipeline compiler + // + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_NO_PROXY", "NO_PROXY", "no_proxy"}, + Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps", + Name: "backend-no-proxy", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_HTTP_PROXY", "HTTP_PROXY", "http_proxy"}, + Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps", + Name: "backend-http-proxy", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_HTTPS_PROXY", "HTTPS_PROXY", "https_proxy"}, + Usage: "if set, pass the environment variable down as \"HTTPS_PROXY\" to steps", + Name: "backend-https-proxy", + }, + // // Please note the below flags should match the flags from // pipeline/frontend/metadata.go and should be kept synchronized. @@ -292,87 +311,4 @@ var flags = []cli.Flag{ EnvVars: []string{"CI_FORGE_URL"}, Name: "forge-url", }, - - // backend docker - &cli.BoolFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_ENABLE_IPV6"}, - Name: "backend-docker-ipv6", - Usage: "backend docker enable IPV6", - Value: false, - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_NETWORK"}, - Name: "backend-docker-network", - Usage: "backend docker network", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_VOLUMES"}, - Name: "backend-docker-volumes", - Usage: "backend docker volumes (comma separated)", - }, - - // backend ssh - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_SSH_ADDRESS"}, - Name: "backend-ssh-address", - Usage: "backend ssh address", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_SSH_USER"}, - Name: "backend-ssh-user", - Usage: "backend ssh user", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY"}, - Name: "backend-ssh-key", - Usage: "backend ssh key file", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY_PASSWORD"}, - Name: "backend-ssh-key-password", - Usage: "backend ssh key password", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_SSH_PASSWORD"}, - Name: "backend-ssh-password", - Usage: "backend ssh password", - }, - - // backend k8s - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_NAMESPACE"}, - Name: "backend-k8s-namespace", - Usage: "backend k8s namespace", - Value: "woodpecker", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_VOLUME_SIZE"}, - Name: "backend-k8s-volume-size", - Usage: "backend k8s volume size (default 10G)", - Value: "10G", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_CLASS"}, - Name: "backend-k8s-storage-class", - Usage: "backend k8s storage class", - Value: "", - }, - &cli.BoolFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_RWX"}, - Name: "backend-k8s-storage-rwx", - Usage: "backend k8s storage access mode, should ReadWriteMany (RWX) instead of ReadWriteOnce (RWO) be used? (default: true)", - Value: true, - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"}, - Name: "backend-k8s-pod-labels", - Usage: "backend k8s additional worker pod labels", - Value: "", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"}, - Name: "backend-k8s-pod-annotations", - Usage: "backend k8s additional worker pod annotations", - Value: "", - }, } diff --git a/cmd/agent/flags.go b/cmd/agent/flags.go index 4d70fb316..481afd486 100644 --- a/cmd/agent/flags.go +++ b/cmd/agent/flags.go @@ -20,11 +20,9 @@ import ( "time" "github.com/urfave/cli/v2" - - "github.com/woodpecker-ci/woodpecker/cmd/common" ) -var flags = append([]cli.Flag{ +var flags = []cli.Flag{ &cli.StringFlag{ EnvVars: []string{"WOODPECKER_SERVER"}, Name: "server", @@ -99,99 +97,4 @@ var flags = append([]cli.Flag{ Usage: "backend engine to run pipelines on", Value: "auto-detect", }, - - // backend docker - &cli.BoolFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_ENABLE_IPV6"}, - Name: "backend-docker-ipv6", - Usage: "backend docker enable IPV6", - Value: false, - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_NETWORK"}, - Name: "backend-docker-network", - Usage: "backend docker network", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_VOLUMES"}, - Name: "backend-docker-volumes", - Usage: "backend docker volumes (comma separated)", - }, - - // backend ssh - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_SSH_ADDRESS"}, - Name: "backend-ssh-address", - Usage: "backend ssh address", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_SSH_USER"}, - Name: "backend-ssh-user", - Usage: "backend ssh user", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY"}, - Name: "backend-ssh-key", - Usage: "backend ssh key file", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY_PASSWORD"}, - Name: "backend-ssh-key-password", - Usage: "backend ssh key password", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_SSH_PASSWORD"}, - Name: "backend-ssh-password", - Usage: "backend ssh password", - }, - - // backend k8s - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_NAMESPACE"}, - Name: "backend-k8s-namespace", - Usage: "backend k8s namespace", - Value: "woodpecker", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_VOLUME_SIZE"}, - Name: "backend-k8s-volume-size", - Usage: "backend k8s volume size (default 10G)", - Value: "10G", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_CLASS"}, - Name: "backend-k8s-storage-class", - Usage: "backend k8s storage class", - Value: "", - }, - &cli.BoolFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_RWX"}, - Name: "backend-k8s-storage-rwx", - Usage: "backend k8s storage access mode, should ReadWriteMany (RWX) instead of ReadWriteOnce (RWO) be used? (default: true)", - Value: true, - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"}, - Name: "backend-k8s-pod-labels", - Usage: "backend k8s additional worker pod labels", - Value: "", - }, - &cli.StringFlag{ - EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"}, - Name: "backend-k8s-pod-annotations", - Usage: "backend k8s additional worker pod annotations", - Value: "", - }, - &cli.IntFlag{ - EnvVars: []string{"WOODPECKER_CONNECT_RETRY_COUNT"}, - Name: "connect-retry-count", - Usage: "number of times to retry connecting to the server", - Value: 5, - }, - &cli.DurationFlag{ - EnvVars: []string{"WOODPECKER_CONNECT_RETRY_DELAY"}, - Name: "connect-retry-delay", - Usage: "duration to wait before retrying to connect to the server", - Value: time.Second * 2, - }, -}, common.GlobalLoggerFlags...) +} diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 7543f87fb..bed58fc44 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -21,6 +21,12 @@ import ( _ "github.com/joho/godotenv/autoload" "github.com/urfave/cli/v2" + "github.com/woodpecker-ci/woodpecker/cmd/common" + "github.com/woodpecker-ci/woodpecker/pipeline/backend/docker" + "github.com/woodpecker-ci/woodpecker/pipeline/backend/kubernetes" + "github.com/woodpecker-ci/woodpecker/pipeline/backend/local" + "github.com/woodpecker-ci/woodpecker/pipeline/backend/ssh" + "github.com/woodpecker-ci/woodpecker/shared/utils" "github.com/woodpecker-ci/woodpecker/version" ) @@ -37,7 +43,7 @@ func main() { Action: pinger, }, } - app.Flags = flags + app.Flags = utils.MergeSlices(flags, common.GlobalLoggerFlags, docker.Flags, ssh.Flags, kubernetes.Flags, local.Flags) if err := app.Run(os.Args); err != nil { fmt.Fprintln(os.Stderr, err) diff --git a/cmd/server/flags.go b/cmd/server/flags.go index 6e9eef462..c4c4ef552 100644 --- a/cmd/server/flags.go +++ b/cmd/server/flags.go @@ -263,6 +263,24 @@ var flags = append([]cli.Flag{ Value: true, }, // + // backend options for pipeline compiler + // + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_NO_PROXY", "NO_PROXY", "no_proxy"}, + Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps", + Name: "backend-no-proxy", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_HTTP_PROXY", "HTTP_PROXY", "http_proxy"}, + Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps", + Name: "backend-http-proxy", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_HTTPS_PROXY", "HTTPS_PROXY", "https_proxy"}, + Usage: "if set, pass the environment variable down as \"HTTPS_PROXY\" to steps", + Name: "backend-https-proxy", + }, + // // resource limit parameters // &cli.DurationFlag{ diff --git a/cmd/server/server.go b/cmd/server/server.go index 841278e09..719669fe5 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -318,6 +318,11 @@ func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) { server.Config.Pipeline.Limits.CPUShares = c.Int64("limit-cpu-shares") server.Config.Pipeline.Limits.CPUSet = c.String("limit-cpu-set") + // backend options for pipeline compiler + server.Config.Pipeline.Proxy.No = c.String("backend-no-proxy") + server.Config.Pipeline.Proxy.HTTP = c.String("backend-http-proxy") + server.Config.Pipeline.Proxy.HTTPS = c.String("backend-https-proxy") + // server configuration server.Config.Server.Cert = c.String("server-cert") server.Config.Server.Key = c.String("server-key") diff --git a/pipeline/backend/docker/docker.go b/pipeline/backend/docker/docker.go index a2d365bee..9edb03ee7 100644 --- a/pipeline/backend/docker/docker.go +++ b/pipeline/backend/docker/docker.go @@ -17,13 +17,16 @@ package docker import ( "context" "io" + "net/http" "os" + "path/filepath" "runtime" "strings" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/volume" + "github.com/docker/go-connections/tlsconfig" "github.com/moby/moby/client" "github.com/moby/moby/pkg/jsonmessage" "github.com/moby/moby/pkg/stdcopy" @@ -67,20 +70,56 @@ func (e *docker) IsAvailable(context.Context) bool { return err == nil } +func httpClientOfOpts(dockerCertPath string, verifyTLS bool) *http.Client { + if dockerCertPath == "" { + return nil + } + + options := tlsconfig.Options{ + CAFile: filepath.Join(dockerCertPath, "ca.pem"), + CertFile: filepath.Join(dockerCertPath, "cert.pem"), + KeyFile: filepath.Join(dockerCertPath, "key.pem"), + InsecureSkipVerify: !verifyTLS, + } + tlsConf, err := tlsconfig.Client(options) + if err != nil { + log.Error().Err(err).Msg("could not create http client out of docker backend options") + return nil + } + + return &http.Client{ + Transport: &http.Transport{TLSClientConfig: tlsConf}, + CheckRedirect: client.CheckRedirect, + } +} + // Load new client for Docker Engine using environment variables. func (e *docker) Load(ctx context.Context) error { - cl, err := client.NewClientWithOpts(client.FromEnv) + c, ok := ctx.Value(backend.CliContext).(*cli.Context) + if !ok { + return backend.ErrNoCliContextFound + } + + var dockerClientOpts []client.Opt + if httpClient := httpClientOfOpts(c.String("backend-docker-cert"), c.Bool("backend-docker-tls-verify")); httpClient != nil { + dockerClientOpts = append(dockerClientOpts, client.WithHTTPClient(httpClient)) + } + if dockerHost := c.String("backend-docker-host"); dockerHost != "" { + dockerClientOpts = append(dockerClientOpts, client.WithHost(dockerHost)) + } + if dockerAPIVersion := c.String("backend-docker-api-version"); dockerAPIVersion != "" { + dockerClientOpts = append(dockerClientOpts, client.WithVersion(dockerAPIVersion)) + } else { + dockerClientOpts = append(dockerClientOpts, client.WithAPIVersionNegotiation()) + } + + cl, err := client.NewClientWithOpts(dockerClientOpts...) if err != nil { return err } e.client = cl - c, ok := ctx.Value(backend.CliContext).(*cli.Context) - if !ok { - return backend.ErrNoCliContextFound - } e.enableIPv6 = c.Bool("backend-docker-ipv6") - e.network = c.String("backend-docker-network") volumes := strings.Split(c.String("backend-docker-volumes"), ",") diff --git a/pipeline/backend/docker/flags.go b/pipeline/backend/docker/flags.go new file mode 100644 index 000000000..293bbdfba --- /dev/null +++ b/pipeline/backend/docker/flags.go @@ -0,0 +1,59 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker + +import ( + "github.com/urfave/cli/v2" +) + +var Flags = []cli.Flag{ + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_HOST", "DOCKER_HOST"}, + Name: "backend-docker-host", + Usage: "path to docker socket or url to the docker server", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_API_VERSION", "DOCKER_API_VERSION"}, + Name: "backend-docker-api-version", + Usage: "the version of the API to reach, leave empty for latest.", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_CERT_PATH", "DOCKER_CERT_PATH"}, + Name: "backend-docker-cert", + Usage: "path to load the TLS certificates for connecting to docker server", + }, + &cli.BoolFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_TLS_VERIFY", "DOCKER_TLS_VERIFY"}, + Name: "backend-docker-tls-verify", + Usage: "enable or disable TLS verification for connecting to docker server", + Value: true, + }, + &cli.BoolFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_ENABLE_IPV6"}, + Name: "backend-docker-ipv6", + Usage: "backend docker enable IPV6", + Value: false, + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_NETWORK"}, + Name: "backend-docker-network", + Usage: "backend docker network", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_VOLUMES"}, + Name: "backend-docker-volumes", + Usage: "backend docker volumes (comma separated)", + }, +} diff --git a/pipeline/backend/kubernetes/flags.go b/pipeline/backend/kubernetes/flags.go new file mode 100644 index 000000000..7a51e4291 --- /dev/null +++ b/pipeline/backend/kubernetes/flags.go @@ -0,0 +1,72 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "time" + + "github.com/urfave/cli/v2" +) + +var Flags = []cli.Flag{ + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_NAMESPACE"}, + Name: "backend-k8s-namespace", + Usage: "backend k8s namespace", + Value: "woodpecker", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_VOLUME_SIZE"}, + Name: "backend-k8s-volume-size", + Usage: "backend k8s volume size (default 10G)", + Value: "10G", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_CLASS"}, + Name: "backend-k8s-storage-class", + Usage: "backend k8s storage class", + Value: "", + }, + &cli.BoolFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_RWX"}, + Name: "backend-k8s-storage-rwx", + Usage: "backend k8s storage access mode, should ReadWriteMany (RWX) instead of ReadWriteOnce (RWO) be used? (default: true)", + Value: true, + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"}, + Name: "backend-k8s-pod-labels", + Usage: "backend k8s additional worker pod labels", + Value: "", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"}, + Name: "backend-k8s-pod-annotations", + Usage: "backend k8s additional worker pod annotations", + Value: "", + }, + &cli.IntFlag{ + EnvVars: []string{"WOODPECKER_CONNECT_RETRY_COUNT"}, + Name: "connect-retry-count", + Usage: "number of times to retry connecting to the server", + Value: 5, + }, + &cli.DurationFlag{ + EnvVars: []string{"WOODPECKER_CONNECT_RETRY_DELAY"}, + Name: "connect-retry-delay", + Usage: "duration to wait before retrying to connect to the server", + Value: time.Second * 2, + }, +} diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index aba979a52..8b23562a0 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -1,3 +1,17 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package kubernetes import ( diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index 1ab91478a..da2a3e58a 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -1,3 +1,17 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package kubernetes import ( diff --git a/pipeline/backend/kubernetes/service.go b/pipeline/backend/kubernetes/service.go index 6ed0eb108..224bc7c6c 100644 --- a/pipeline/backend/kubernetes/service.go +++ b/pipeline/backend/kubernetes/service.go @@ -1,3 +1,17 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package kubernetes import ( diff --git a/pipeline/backend/kubernetes/service_test.go b/pipeline/backend/kubernetes/service_test.go index 82a952c83..6702b36d5 100644 --- a/pipeline/backend/kubernetes/service_test.go +++ b/pipeline/backend/kubernetes/service_test.go @@ -1,3 +1,17 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package kubernetes import ( diff --git a/pipeline/backend/kubernetes/utils.go b/pipeline/backend/kubernetes/utils.go index 50801febf..3b177e2fc 100644 --- a/pipeline/backend/kubernetes/utils.go +++ b/pipeline/backend/kubernetes/utils.go @@ -1,3 +1,17 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package kubernetes import ( @@ -41,13 +55,13 @@ func isImagePullBackOffState(pod *v1.Pod) bool { // getClientOutOfCluster returns a k8s clientset to the request from outside of cluster func getClientOutOfCluster() (kubernetes.Interface, error) { - kubeconfigPath := os.Getenv("KUBECONFIG") - if kubeconfigPath == "" { - kubeconfigPath = os.Getenv("HOME") + "/.kube/config" + kubeConfigPath := os.Getenv("KUBECONFIG") + if kubeConfigPath == "" { + kubeConfigPath = os.Getenv("HOME") + "/.kube/config" } // use the current context in kubeconfig - config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) if err != nil { return nil, err } diff --git a/pipeline/backend/kubernetes/volume.go b/pipeline/backend/kubernetes/volume.go index 12ebd9cb7..d8c9ee72d 100644 --- a/pipeline/backend/kubernetes/volume.go +++ b/pipeline/backend/kubernetes/volume.go @@ -1,3 +1,17 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package kubernetes import ( diff --git a/pipeline/backend/kubernetes/volume_test.go b/pipeline/backend/kubernetes/volume_test.go index 0a8cc8f81..d1132c46f 100644 --- a/pipeline/backend/kubernetes/volume_test.go +++ b/pipeline/backend/kubernetes/volume_test.go @@ -1,3 +1,17 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package kubernetes import ( diff --git a/pipeline/backend/local/flags.go b/pipeline/backend/local/flags.go new file mode 100644 index 000000000..9c67ac89b --- /dev/null +++ b/pipeline/backend/local/flags.go @@ -0,0 +1,21 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local + +import ( + "github.com/urfave/cli/v2" +) + +var Flags = []cli.Flag{} diff --git a/pipeline/backend/ssh/flags.go b/pipeline/backend/ssh/flags.go new file mode 100644 index 000000000..630351cba --- /dev/null +++ b/pipeline/backend/ssh/flags.go @@ -0,0 +1,47 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ssh + +import ( + "github.com/urfave/cli/v2" +) + +var Flags = []cli.Flag{ + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_SSH_ADDRESS"}, + Name: "backend-ssh-address", + Usage: "backend ssh address", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_SSH_USER"}, + Name: "backend-ssh-user", + Usage: "backend ssh user", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY"}, + Name: "backend-ssh-key", + Usage: "backend ssh key file", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY_PASSWORD"}, + Name: "backend-ssh-key-password", + Usage: "backend ssh key password", + }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_SSH_PASSWORD"}, + Name: "backend-ssh-password", + Usage: "backend ssh password", + }, +} diff --git a/pipeline/backend/ssh/ssh.go b/pipeline/backend/ssh/ssh.go index f22d7d7a1..ed83960b3 100644 --- a/pipeline/backend/ssh/ssh.go +++ b/pipeline/backend/ssh/ssh.go @@ -1,3 +1,17 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ssh import ( diff --git a/pipeline/frontend/yaml/compiler/option.go b/pipeline/frontend/yaml/compiler/option.go index 055b38e3b..65d9908f9 100644 --- a/pipeline/frontend/yaml/compiler/option.go +++ b/pipeline/frontend/yaml/compiler/option.go @@ -16,7 +16,6 @@ package compiler import ( "net/url" - "os" "path/filepath" "strings" @@ -174,22 +173,6 @@ func WithS3Cacher(access, secret, region, bucket string) Option { } } -// WithProxy configures the compiler with HTTP_PROXY, HTTPS_PROXY, -// and NO_PROXY environment variables added by default to every -// container in the pipeline. -func WithProxy() Option { - return WithEnviron( - map[string]string{ - "no_proxy": noProxy, - "NO_PROXY": noProxy, - "http_proxy": httpProxy, - "HTTP_PROXY": httpProxy, - "HTTPS_PROXY": httpsProxy, - "https_proxy": httpsProxy, - }, - ) -} - // WithNetworks configures the compiler with additional networks // to be connected to pipeline containers func WithNetworks(networks ...string) Option { @@ -233,38 +216,24 @@ func WithNetrcOnlyTrusted(only bool) Option { } } -// TODO(bradrydzewski) consider an alternate approach to -// WithProxy where the proxy strings are passed directly -// to the function as named parameters. - -// func WithProxy2(http, https, none string) Option { -// return WithEnviron( -// map[string]string{ -// "no_proxy": none, -// "NO_PROXY": none, -// "http_proxy": http, -// "HTTP_PROXY": http, -// "HTTPS_PROXY": https, -// "https_proxy": https, -// }, -// ) -// } - -var ( - noProxy = getenv("no_proxy") - httpProxy = getenv("https_proxy") - httpsProxy = getenv("https_proxy") -) - -// getenv returns the named environment variable. -func getenv(name string) (value string) { - name = strings.ToUpper(name) - if value := os.Getenv(name); value != "" { - return value - } - name = strings.ToLower(name) - if value := os.Getenv(name); value != "" { - return value - } - return +type ProxyOptions struct { + NoProxy string + HTTPProxy string + HTTPSProxy string +} + +// WithProxy configures the compiler with HTTP_PROXY, HTTPS_PROXY, +// and NO_PROXY environment variables added by default to every +// container in the pipeline. +func WithProxy(opt ProxyOptions) Option { + return WithEnviron( + map[string]string{ + "no_proxy": opt.NoProxy, + "NO_PROXY": opt.NoProxy, + "http_proxy": opt.HTTPProxy, + "HTTP_PROXY": opt.HTTPProxy, + "HTTPS_PROXY": opt.HTTPSProxy, + "https_proxy": opt.HTTPSProxy, + }, + ) } diff --git a/pipeline/frontend/yaml/compiler/option_test.go b/pipeline/frontend/yaml/compiler/option_test.go index 3af0acddb..706b70199 100644 --- a/pipeline/frontend/yaml/compiler/option_test.go +++ b/pipeline/frontend/yaml/compiler/option_test.go @@ -1,7 +1,6 @@ package compiler import ( - "os" "reflect" "testing" @@ -153,24 +152,10 @@ func TestWithNetrc(t *testing.T) { } func TestWithProxy(t *testing.T) { - // do not execute the test if the host machine sets http proxy - // environment variables to avoid interference with other tests. - if noProxy != "" || httpProxy != "" || httpsProxy != "" { - t.SkipNow() - return - } - // alter the default values - noProxy = "example.com" - httpProxy = "bar.com" - httpsProxy = "baz.com" - - // reset the default values - defer func() { - noProxy = "" - httpProxy = "" - httpsProxy = "" - }() + noProxy := "example.com" + httpProxy := "bar.com" + httpsProxy := "baz.com" testdata := map[string]string{ "no_proxy": noProxy, @@ -181,7 +166,11 @@ func TestWithProxy(t *testing.T) { "HTTPS_PROXY": httpsProxy, } compiler := New( - WithProxy(), + WithProxy(ProxyOptions{ + NoProxy: noProxy, + HTTPProxy: httpProxy, + HTTPSProxy: httpsProxy, + }), ) for key, value := range testdata { if compiler.env[key] != value { @@ -207,26 +196,6 @@ func TestWithEnviron(t *testing.T) { } } -func TestGetenv(t *testing.T) { - defer func() { - os.Unsetenv("X_TEST_FOO") - os.Unsetenv("x_test_bar") - os.Unsetenv("x_test_baz") - }() - os.Setenv("X_TEST_FOO", "foo") - os.Setenv("x_test_bar", "bar") - os.Setenv("x_test_baz", "") - if getenv("x_test_foo") != "foo" { - t.Errorf("Expect X_TEST_FOO=foo") - } - if getenv("X_TEST_BAR") != "bar" { - t.Errorf("Expect x_test_bar=bar") - } - if getenv("x_test_baz") != "" { - t.Errorf("Expect x_test_bar=bar is empty") - } -} - func TestWithVolumeCacher(t *testing.T) { compiler := New( WithVolumeCacher("/cache"), diff --git a/pipeline/stepBuilder.go b/pipeline/stepBuilder.go index 2def2fadd..a8cc86d2b 100644 --- a/pipeline/stepBuilder.go +++ b/pipeline/stepBuilder.go @@ -39,16 +39,17 @@ import ( // StepBuilder Takes the hook data and the yaml and returns in internal data model type StepBuilder struct { - Repo *model.Repo - Curr *model.Pipeline - Last *model.Pipeline - Netrc *model.Netrc - Secs []*model.Secret - Regs []*model.Registry - Link string - Yamls []*forge_types.FileMeta - Envs map[string]string - Forge metadata.ServerForge + Repo *model.Repo + Curr *model.Pipeline + Last *model.Pipeline + Netrc *model.Netrc + Secs []*model.Secret + Regs []*model.Registry + Link string + Yamls []*forge_types.FileMeta + Envs map[string]string + Forge metadata.ServerForge + ProxyOpts compiler.ProxyOptions } type Item struct { @@ -281,7 +282,7 @@ func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, envi stepID, ), ), - compiler.WithProxy(), + compiler.WithProxy(b.ProxyOpts), compiler.WithWorkspaceFromURL("/woodpecker", b.Repo.Link), compiler.WithMetadata(metadata), compiler.WithTrusted(b.Repo.IsTrusted), diff --git a/server/config.go b/server/config.go index 94edad8d1..59a1b73e4 100644 --- a/server/config.go +++ b/server/config.go @@ -91,5 +91,10 @@ var Config = struct { Privileged []string DefaultTimeout int64 MaxTimeout int64 + Proxy struct { + No string + HTTP string + HTTPS string + } } }{} diff --git a/server/pipeline/items.go b/server/pipeline/items.go index 6c0b67cc5..1eca77b49 100644 --- a/server/pipeline/items.go +++ b/server/pipeline/items.go @@ -22,6 +22,7 @@ import ( "github.com/rs/zerolog/log" "github.com/woodpecker-ci/woodpecker/pipeline" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler" "github.com/woodpecker-ci/woodpecker/server" forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types" "github.com/woodpecker-ci/woodpecker/server/model" @@ -78,6 +79,11 @@ func createPipelineItems(c context.Context, store store.Store, Link: server.Config.Server.Host, Yamls: yamls, Forge: server.Config.Services.Forge, + ProxyOpts: compiler.ProxyOptions{ + NoProxy: server.Config.Pipeline.Proxy.No, + HTTPProxy: server.Config.Pipeline.Proxy.HTTP, + HTTPSProxy: server.Config.Pipeline.Proxy.HTTPS, + }, } pipelineItems, err := b.Build() if err != nil { diff --git a/shared/utils/paginate.go b/shared/utils/paginate.go index bf8178f88..a8b883ff0 100644 --- a/shared/utils/paginate.go +++ b/shared/utils/paginate.go @@ -1,3 +1,17 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils // Paginate iterates over a func call until it does not return new items and return it as list diff --git a/shared/utils/paginate_test.go b/shared/utils/paginate_test.go index cc675514d..870aba079 100644 --- a/shared/utils/paginate_test.go +++ b/shared/utils/paginate_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils import ( diff --git a/shared/utils/slices.go b/shared/utils/slices.go new file mode 100644 index 000000000..0546b5674 --- /dev/null +++ b/shared/utils/slices.go @@ -0,0 +1,33 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +// MergeSlices return a new slice that combines all values of input slices +// TODO: once https://github.com/golang/go/pull/61817 got merged, we should switch to it +func MergeSlices[T any](slices ...[]T) []T { + sl := 0 + for i := range slices { + sl += len(slices[i]) + } + result := make([]T, sl) + cp := 0 + for _, s := range slices { + if sLen := len(s); sLen != 0 { + copy(result[cp:], s) + cp += sLen + } + } + return result +} diff --git a/shared/utils/slices_test.go b/shared/utils/slices_test.go new file mode 100644 index 000000000..80e3feb57 --- /dev/null +++ b/shared/utils/slices_test.go @@ -0,0 +1,29 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMergeSlices(t *testing.T) { + resultSS := MergeSlices([]string{}, []string{"a", "b"}, []string{"c"}, nil) + assert.EqualValues(t, []string{"a", "b", "c"}, resultSS) + + resultIS := MergeSlices([]int{}, []int{1, 2}, []int{4}, nil) + assert.EqualValues(t, []int{1, 2, 4}, resultIS) +}