self-hosted/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-11-24 08:02:18 +02:00
Code Issues Releases Activity

Make sure we dont have hidden options for backend and pipeline compiler (#2123)

Browse Source 
move options based on **os.Getenv** into flags

---------
*Sponsored by Kithara Software GmbH*
This commit is contained in:
6543 2023-08-07 21:13:26 +02:00 committed by GitHub
parent 3d4758578a
commit d253f8cc30
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 562 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cli/exec/exec.go
View File

@ -30,6 +30,10 @@ import (
"github.com/woodpecker-ci/woodpecker/cli/common"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/kubernetes"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/local"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/ssh"
backendTypes "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler"
@ -45,7 +49,7 @@ var Command = &cli.Command{
Usage: "execute a local pipeline",
ArgsUsage: "[path/to/.woodpecker.yaml]",
Action: run,
Flags: append(common.GlobalFlags, flags...),
Flags: utils.MergeSlices(common.GlobalFlags, flags, docker.Flags, ssh.Flags, kubernetes.Flags, local.Flags),
}
func run(c *cli.Context) error {
@ -184,7 +188,11 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error
compiler.WithPrefix(
c.String("prefix"),
),
compiler.WithProxy(),
compiler.WithProxy(compiler.ProxyOptions{
NoProxy: c.String("backend-no-proxy"),
HTTPProxy: c.String("backend-http-proxy"),
HTTPSProxy: c.String("backend-https-proxy"),
}),
compiler.WithLocal(
c.Bool("local"),
),

102
cli/exec/flags.go
View File

@ -64,6 +64,25 @@ var flags = []cli.Flag{
Value: "auto-detect",
},
//
// backend options for pipeline compiler
//
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_NO_PROXY", "NO_PROXY", "no_proxy"},
Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps",
Name: "backend-no-proxy",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_HTTP_PROXY", "HTTP_PROXY", "http_proxy"},
Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps",
Name: "backend-http-proxy",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_HTTPS_PROXY", "HTTPS_PROXY", "https_proxy"},
Usage: "if set, pass the environment variable down as \"HTTPS_PROXY\" to steps",
Name: "backend-https-proxy",
},
//
// Please note the below flags should match the flags from
// pipeline/frontend/metadata.go and should be kept synchronized.
@ -292,87 +311,4 @@ var flags = []cli.Flag{
EnvVars: []string{"CI_FORGE_URL"},
Name: "forge-url",
},
// backend docker
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_ENABLE_IPV6"},
Name: "backend-docker-ipv6",
Usage: "backend docker enable IPV6",
Value: false,
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_NETWORK"},
Name: "backend-docker-network",
Usage: "backend docker network",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_VOLUMES"},
Name: "backend-docker-volumes",
Usage: "backend docker volumes (comma separated)",
},
// backend ssh
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_ADDRESS"},
Name: "backend-ssh-address",
Usage: "backend ssh address",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_USER"},
Name: "backend-ssh-user",
Usage: "backend ssh user",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY"},
Name: "backend-ssh-key",
Usage: "backend ssh key file",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY_PASSWORD"},
Name: "backend-ssh-key-password",
Usage: "backend ssh key password",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_PASSWORD"},
Name: "backend-ssh-password",
Usage: "backend ssh password",
},
// backend k8s
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_NAMESPACE"},
Name: "backend-k8s-namespace",
Usage: "backend k8s namespace",
Value: "woodpecker",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_VOLUME_SIZE"},
Name: "backend-k8s-volume-size",
Usage: "backend k8s volume size (default 10G)",
Value: "10G",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_CLASS"},
Name: "backend-k8s-storage-class",
Usage: "backend k8s storage class",
Value: "",
},
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_RWX"},
Name: "backend-k8s-storage-rwx",
Usage: "backend k8s storage access mode, should ReadWriteMany (RWX) instead of ReadWriteOnce (RWO) be used? (default: true)",
Value: true,
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"},
Name: "backend-k8s-pod-labels",
Usage: "backend k8s additional worker pod labels",
Value: "",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"},
Name: "backend-k8s-pod-annotations",
Usage: "backend k8s additional worker pod annotations",
Value: "",
},
}

101
cmd/agent/flags.go
View File

@ -20,11 +20,9 @@ import (
"time"
"github.com/urfave/cli/v2"
"github.com/woodpecker-ci/woodpecker/cmd/common"
)
var flags = append([]cli.Flag{
var flags = []cli.Flag{
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_SERVER"},
Name: "server",
@ -99,99 +97,4 @@ var flags = append([]cli.Flag{
Usage: "backend engine to run pipelines on",
Value: "auto-detect",
},
// backend docker
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_ENABLE_IPV6"},
Name: "backend-docker-ipv6",
Usage: "backend docker enable IPV6",
Value: false,
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_NETWORK"},
Name: "backend-docker-network",
Usage: "backend docker network",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_VOLUMES"},
Name: "backend-docker-volumes",
Usage: "backend docker volumes (comma separated)",
},
// backend ssh
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_ADDRESS"},
Name: "backend-ssh-address",
Usage: "backend ssh address",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_USER"},
Name: "backend-ssh-user",
Usage: "backend ssh user",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY"},
Name: "backend-ssh-key",
Usage: "backend ssh key file",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY_PASSWORD"},
Name: "backend-ssh-key-password",
Usage: "backend ssh key password",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_PASSWORD"},
Name: "backend-ssh-password",
Usage: "backend ssh password",
},
// backend k8s
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_NAMESPACE"},
Name: "backend-k8s-namespace",
Usage: "backend k8s namespace",
Value: "woodpecker",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_VOLUME_SIZE"},
Name: "backend-k8s-volume-size",
Usage: "backend k8s volume size (default 10G)",
Value: "10G",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_CLASS"},
Name: "backend-k8s-storage-class",
Usage: "backend k8s storage class",
Value: "",
},
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_RWX"},
Name: "backend-k8s-storage-rwx",
Usage: "backend k8s storage access mode, should ReadWriteMany (RWX) instead of ReadWriteOnce (RWO) be used? (default: true)",
Value: true,
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"},
Name: "backend-k8s-pod-labels",
Usage: "backend k8s additional worker pod labels",
Value: "",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"},
Name: "backend-k8s-pod-annotations",
Usage: "backend k8s additional worker pod annotations",
Value: "",
},
&cli.IntFlag{
EnvVars: []string{"WOODPECKER_CONNECT_RETRY_COUNT"},
Name: "connect-retry-count",
Usage: "number of times to retry connecting to the server",
Value: 5,
},
&cli.DurationFlag{
EnvVars: []string{"WOODPECKER_CONNECT_RETRY_DELAY"},
Name: "connect-retry-delay",
Usage: "duration to wait before retrying to connect to the server",
Value: time.Second * 2,
},
}, common.GlobalLoggerFlags...)
}

8
cmd/agent/main.go
View File

@ -21,6 +21,12 @@ import (
_ "github.com/joho/godotenv/autoload"
"github.com/urfave/cli/v2"
"github.com/woodpecker-ci/woodpecker/cmd/common"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/kubernetes"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/local"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/ssh"
"github.com/woodpecker-ci/woodpecker/shared/utils"
"github.com/woodpecker-ci/woodpecker/version"
)
@ -37,7 +43,7 @@ func main() {
Action: pinger,
},
}
app.Flags = flags
app.Flags = utils.MergeSlices(flags, common.GlobalLoggerFlags, docker.Flags, ssh.Flags, kubernetes.Flags, local.Flags)
if err := app.Run(os.Args); err != nil {
fmt.Fprintln(os.Stderr, err)

18
cmd/server/flags.go
View File

@ -263,6 +263,24 @@ var flags = append([]cli.Flag{
Value: true,
},
//
// backend options for pipeline compiler
//
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_NO_PROXY", "NO_PROXY", "no_proxy"},
Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps",
Name: "backend-no-proxy",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_HTTP_PROXY", "HTTP_PROXY", "http_proxy"},
Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps",
Name: "backend-http-proxy",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_HTTPS_PROXY", "HTTPS_PROXY", "https_proxy"},
Usage: "if set, pass the environment variable down as \"HTTPS_PROXY\" to steps",
Name: "backend-https-proxy",
},
//
// resource limit parameters
//
&cli.DurationFlag{

5
cmd/server/server.go
View File

@ -318,6 +318,11 @@ func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) {
server.Config.Pipeline.Limits.CPUShares = c.Int64("limit-cpu-shares")
server.Config.Pipeline.Limits.CPUSet = c.String("limit-cpu-set")
// backend options for pipeline compiler
server.Config.Pipeline.Proxy.No = c.String("backend-no-proxy")
server.Config.Pipeline.Proxy.HTTP = c.String("backend-http-proxy")
server.Config.Pipeline.Proxy.HTTPS = c.String("backend-https-proxy")
// server configuration
server.Config.Server.Cert = c.String("server-cert")
server.Config.Server.Key = c.String("server-key")

51
pipeline/backend/docker/docker.go
View File

@ -17,13 +17,16 @@ package docker
import (
"context"
"io"
"net/http"
"os"
"path/filepath"
"runtime"
"strings"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/api/types/volume"
"github.com/docker/go-connections/tlsconfig"
"github.com/moby/moby/client"
"github.com/moby/moby/pkg/jsonmessage"
"github.com/moby/moby/pkg/stdcopy"
@ -67,20 +70,56 @@ func (e *docker) IsAvailable(context.Context) bool {
return err == nil
}
func httpClientOfOpts(dockerCertPath string, verifyTLS bool) *http.Client {
if dockerCertPath == "" {
return nil
}
options := tlsconfig.Options{
CAFile: filepath.Join(dockerCertPath, "ca.pem"),
CertFile: filepath.Join(dockerCertPath, "cert.pem"),
KeyFile: filepath.Join(dockerCertPath, "key.pem"),
InsecureSkipVerify: !verifyTLS,
}
tlsConf, err := tlsconfig.Client(options)
if err != nil {
log.Error().Err(err).Msg("could not create http client out of docker backend options")
return nil
}
return &http.Client{
Transport: &http.Transport{TLSClientConfig: tlsConf},
CheckRedirect: client.CheckRedirect,
}
}
// Load new client for Docker Engine using environment variables.
func (e *docker) Load(ctx context.Context) error {
cl, err := client.NewClientWithOpts(client.FromEnv)
c, ok := ctx.Value(backend.CliContext).(*cli.Context)
if !ok {
return backend.ErrNoCliContextFound
}
var dockerClientOpts []client.Opt
if httpClient := httpClientOfOpts(c.String("backend-docker-cert"), c.Bool("backend-docker-tls-verify")); httpClient != nil {
dockerClientOpts = append(dockerClientOpts, client.WithHTTPClient(httpClient))
}
if dockerHost := c.String("backend-docker-host"); dockerHost != "" {
dockerClientOpts = append(dockerClientOpts, client.WithHost(dockerHost))
}
if dockerAPIVersion := c.String("backend-docker-api-version"); dockerAPIVersion != "" {
dockerClientOpts = append(dockerClientOpts, client.WithVersion(dockerAPIVersion))
} else {
dockerClientOpts = append(dockerClientOpts, client.WithAPIVersionNegotiation())
}
cl, err := client.NewClientWithOpts(dockerClientOpts...)
if err != nil {
return err
}
e.client = cl
c, ok := ctx.Value(backend.CliContext).(*cli.Context)
if !ok {
return backend.ErrNoCliContextFound
}
e.enableIPv6 = c.Bool("backend-docker-ipv6")
e.network = c.String("backend-docker-network")
volumes := strings.Split(c.String("backend-docker-volumes"), ",")

59
pipeline/backend/docker/flags.go Normal file
View File

@ -0,0 +1,59 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package docker
import (
"github.com/urfave/cli/v2"
)
var Flags = []cli.Flag{
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_HOST", "DOCKER_HOST"},
Name: "backend-docker-host",
Usage: "path to docker socket or url to the docker server",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_API_VERSION", "DOCKER_API_VERSION"},
Name: "backend-docker-api-version",
Usage: "the version of the API to reach, leave empty for latest.",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_CERT_PATH", "DOCKER_CERT_PATH"},
Name: "backend-docker-cert",
Usage: "path to load the TLS certificates for connecting to docker server",
},
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_TLS_VERIFY", "DOCKER_TLS_VERIFY"},
Name: "backend-docker-tls-verify",
Usage: "enable or disable TLS verification for connecting to docker server",
Value: true,
},
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_ENABLE_IPV6"},
Name: "backend-docker-ipv6",
Usage: "backend docker enable IPV6",
Value: false,
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_NETWORK"},
Name: "backend-docker-network",
Usage: "backend docker network",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_VOLUMES"},
Name: "backend-docker-volumes",
Usage: "backend docker volumes (comma separated)",
},
}

72
pipeline/backend/kubernetes/flags.go Normal file
View File

@ -0,0 +1,72 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kubernetes
import (
"time"
"github.com/urfave/cli/v2"
)
var Flags = []cli.Flag{
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_NAMESPACE"},
Name: "backend-k8s-namespace",
Usage: "backend k8s namespace",
Value: "woodpecker",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_VOLUME_SIZE"},
Name: "backend-k8s-volume-size",
Usage: "backend k8s volume size (default 10G)",
Value: "10G",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_CLASS"},
Name: "backend-k8s-storage-class",
Usage: "backend k8s storage class",
Value: "",
},
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_RWX"},
Name: "backend-k8s-storage-rwx",
Usage: "backend k8s storage access mode, should ReadWriteMany (RWX) instead of ReadWriteOnce (RWO) be used? (default: true)",
Value: true,
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"},
Name: "backend-k8s-pod-labels",
Usage: "backend k8s additional worker pod labels",
Value: "",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"},
Name: "backend-k8s-pod-annotations",
Usage: "backend k8s additional worker pod annotations",
Value: "",
},
&cli.IntFlag{
EnvVars: []string{"WOODPECKER_CONNECT_RETRY_COUNT"},
Name: "connect-retry-count",
Usage: "number of times to retry connecting to the server",
Value: 5,
},
&cli.DurationFlag{
EnvVars: []string{"WOODPECKER_CONNECT_RETRY_DELAY"},
Name: "connect-retry-delay",
Usage: "duration to wait before retrying to connect to the server",
Value: time.Second * 2,
},
}

14
pipeline/backend/kubernetes/kubernetes.go
View File

@ -1,3 +1,17 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kubernetes
import (

14
pipeline/backend/kubernetes/pod.go
View File

@ -1,3 +1,17 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kubernetes
import (

14
pipeline/backend/kubernetes/service.go
View File

@ -1,3 +1,17 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kubernetes
import (

14
pipeline/backend/kubernetes/service_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kubernetes
import (

22
pipeline/backend/kubernetes/utils.go
View File

@ -1,3 +1,17 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kubernetes
import (
@ -41,13 +55,13 @@ func isImagePullBackOffState(pod *v1.Pod) bool {
// getClientOutOfCluster returns a k8s clientset to the request from outside of cluster
func getClientOutOfCluster() (kubernetes.Interface, error) {
kubeconfigPath := os.Getenv("KUBECONFIG")
if kubeconfigPath == "" {
kubeconfigPath = os.Getenv("HOME") + "/.kube/config"
kubeConfigPath := os.Getenv("KUBECONFIG")
if kubeConfigPath == "" {
kubeConfigPath = os.Getenv("HOME") + "/.kube/config"
}
// use the current context in kubeconfig
config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
if err != nil {
return nil, err
}

14
pipeline/backend/kubernetes/volume.go
View File

@ -1,3 +1,17 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kubernetes
import (

14
pipeline/backend/kubernetes/volume_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kubernetes
import (

21
pipeline/backend/local/flags.go Normal file
View File

@ -0,0 +1,21 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package local
import (
"github.com/urfave/cli/v2"
)
var Flags = []cli.Flag{}

47
pipeline/backend/ssh/flags.go Normal file
View File

@ -0,0 +1,47 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ssh
import (
"github.com/urfave/cli/v2"
)
var Flags = []cli.Flag{
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_ADDRESS"},
Name: "backend-ssh-address",
Usage: "backend ssh address",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_USER"},
Name: "backend-ssh-user",
Usage: "backend ssh user",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY"},
Name: "backend-ssh-key",
Usage: "backend ssh key file",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY_PASSWORD"},
Name: "backend-ssh-key-password",
Usage: "backend ssh key password",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_PASSWORD"},
Name: "backend-ssh-password",
Usage: "backend ssh password",
},
}

14
pipeline/backend/ssh/ssh.go
View File

@ -1,3 +1,17 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ssh
import (

71
pipeline/frontend/yaml/compiler/option.go
View File

@ -16,7 +16,6 @@ package compiler
import (
"net/url"
"os"
"path/filepath"
"strings"
@ -174,22 +173,6 @@ func WithS3Cacher(access, secret, region, bucket string) Option {
}
}
// WithProxy configures the compiler with HTTP_PROXY, HTTPS_PROXY,
// and NO_PROXY environment variables added by default to every
// container in the pipeline.
func WithProxy() Option {
return WithEnviron(
map[string]string{
"no_proxy": noProxy,
"NO_PROXY": noProxy,
"http_proxy": httpProxy,
"HTTP_PROXY": httpProxy,
"HTTPS_PROXY": httpsProxy,
"https_proxy": httpsProxy,
},
)
}
// WithNetworks configures the compiler with additional networks
// to be connected to pipeline containers
func WithNetworks(networks ...string) Option {
@ -233,38 +216,24 @@ func WithNetrcOnlyTrusted(only bool) Option {
}
}
// TODO(bradrydzewski) consider an alternate approach to
// WithProxy where the proxy strings are passed directly
// to the function as named parameters.
// func WithProxy2(http, https, none string) Option {
// return WithEnviron(
// map[string]string{
// "no_proxy": none,
// "NO_PROXY": none,
// "http_proxy": http,
// "HTTP_PROXY": http,
// "HTTPS_PROXY": https,
// "https_proxy": https,
// },
// )
// }
var (
noProxy = getenv("no_proxy")
httpProxy = getenv("https_proxy")
httpsProxy = getenv("https_proxy")
)
// getenv returns the named environment variable.
func getenv(name string) (value string) {
name = strings.ToUpper(name)
if value := os.Getenv(name); value != "" {
return value
}
name = strings.ToLower(name)
if value := os.Getenv(name); value != "" {
return value
}
return
type ProxyOptions struct {
NoProxy string
HTTPProxy string
HTTPSProxy string
}
// WithProxy configures the compiler with HTTP_PROXY, HTTPS_PROXY,
// and NO_PROXY environment variables added by default to every
// container in the pipeline.
func WithProxy(opt ProxyOptions) Option {
return WithEnviron(
map[string]string{
"no_proxy": opt.NoProxy,
"NO_PROXY": opt.NoProxy,
"http_proxy": opt.HTTPProxy,
"HTTP_PROXY": opt.HTTPProxy,
"HTTPS_PROXY": opt.HTTPSProxy,
"https_proxy": opt.HTTPSProxy,
},
)
}

47
pipeline/frontend/yaml/compiler/option_test.go
View File

@ -1,7 +1,6 @@
package compiler
import (
"os"
"reflect"
"testing"
@ -153,24 +152,10 @@ func TestWithNetrc(t *testing.T) {
}
func TestWithProxy(t *testing.T) {
// do not execute the test if the host machine sets http proxy
// environment variables to avoid interference with other tests.
if noProxy != "" || httpProxy != "" || httpsProxy != "" {
t.SkipNow()
return
}
// alter the default values
noProxy = "example.com"
httpProxy = "bar.com"
httpsProxy = "baz.com"
// reset the default values
defer func() {
noProxy = ""
httpProxy = ""
httpsProxy = ""
}()
noProxy := "example.com"
httpProxy := "bar.com"
httpsProxy := "baz.com"
testdata := map[string]string{
"no_proxy": noProxy,
@ -181,7 +166,11 @@ func TestWithProxy(t *testing.T) {
"HTTPS_PROXY": httpsProxy,
}
compiler := New(
WithProxy(),
WithProxy(ProxyOptions{
NoProxy: noProxy,
HTTPProxy: httpProxy,
HTTPSProxy: httpsProxy,
}),
)
for key, value := range testdata {
if compiler.env[key] != value {
@ -207,26 +196,6 @@ func TestWithEnviron(t *testing.T) {
}
}
func TestGetenv(t *testing.T) {
defer func() {
os.Unsetenv("X_TEST_FOO")
os.Unsetenv("x_test_bar")
os.Unsetenv("x_test_baz")
}()
os.Setenv("X_TEST_FOO", "foo")
os.Setenv("x_test_bar", "bar")
os.Setenv("x_test_baz", "")
if getenv("x_test_foo") != "foo" {
t.Errorf("Expect X_TEST_FOO=foo")
}
if getenv("X_TEST_BAR") != "bar" {
t.Errorf("Expect x_test_bar=bar")
}
if getenv("x_test_baz") != "" {
t.Errorf("Expect x_test_bar=bar is empty")
}
}
func TestWithVolumeCacher(t *testing.T) {
compiler := New(
WithVolumeCacher("/cache"),

23
pipeline/stepBuilder.go
View File

@ -39,16 +39,17 @@ import (
// StepBuilder Takes the hook data and the yaml and returns in internal data model
type StepBuilder struct {
Repo *model.Repo
Curr *model.Pipeline
Last *model.Pipeline
Netrc *model.Netrc
Secs []*model.Secret
Regs []*model.Registry
Link string
Yamls []*forge_types.FileMeta
Envs map[string]string
Forge metadata.ServerForge
Repo *model.Repo
Curr *model.Pipeline
Last *model.Pipeline
Netrc *model.Netrc
Secs []*model.Secret
Regs []*model.Registry
Link string
Yamls []*forge_types.FileMeta
Envs map[string]string
Forge metadata.ServerForge
ProxyOpts compiler.ProxyOptions
}
type Item struct {
@ -281,7 +282,7 @@ func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, envi
stepID,
),
),
compiler.WithProxy(),
compiler.WithProxy(b.ProxyOpts),
compiler.WithWorkspaceFromURL("/woodpecker", b.Repo.Link),
compiler.WithMetadata(metadata),
compiler.WithTrusted(b.Repo.IsTrusted),

5
server/config.go
View File

@ -91,5 +91,10 @@ var Config = struct {
Privileged []string
DefaultTimeout int64
MaxTimeout int64
Proxy struct {
No string
HTTP string
HTTPS string
}
}
}{}

6
server/pipeline/items.go
View File

@ -22,6 +22,7 @@ import (
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler"
"github.com/woodpecker-ci/woodpecker/server"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
@ -78,6 +79,11 @@ func createPipelineItems(c context.Context, store store.Store,
Link: server.Config.Server.Host,
Yamls: yamls,
Forge: server.Config.Services.Forge,
ProxyOpts: compiler.ProxyOptions{
NoProxy: server.Config.Pipeline.Proxy.No,
HTTPProxy: server.Config.Pipeline.Proxy.HTTP,
HTTPSProxy: server.Config.Pipeline.Proxy.HTTPS,
},
}
pipelineItems, err := b.Build()
if err != nil {

14
shared/utils/paginate.go
View File

@ -1,3 +1,17 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
// Paginate iterates over a func call until it does not return new items and return it as list

14
shared/utils/paginate_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (

33
shared/utils/slices.go Normal file
View File

@ -0,0 +1,33 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
// MergeSlices return a new slice that combines all values of input slices
// TODO: once https://github.com/golang/go/pull/61817 got merged, we should switch to it
func MergeSlices[T any](slices ...[]T) []T {
sl := 0
for i := range slices {
sl += len(slices[i])
}
result := make([]T, sl)
cp := 0
for _, s := range slices {
if sLen := len(s); sLen != 0 {
copy(result[cp:], s)
cp += sLen
}
}
return result
}

29
shared/utils/slices_test.go Normal file
View File

@ -0,0 +1,29 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestMergeSlices(t *testing.T) {
resultSS := MergeSlices([]string{}, []string{"a", "b"}, []string{"c"}, nil)
assert.EqualValues(t, []string{"a", "b", "c"}, resultSS)
resultIS := MergeSlices([]int{}, []int{1, 2}, []int{4}, nil)
assert.EqualValues(t, []int{1, 2, 4}, resultIS)
}