feat(kubernetesDeploy): add infrastructure extensibility (#3853)

* feat(kubernetesDeploy): add infrastructure extensibility * update comment * update error handling * remove trailing spaces * chore: refactor to use reuse capabilities * chore: add tests * fix: use proper download function * fix: expose credentials via groovy step * fix: test * chore: remove comment * chore: address CodeClimate findings
2025-11-06 09:09:19 +02:00 · 2022-06-29 12:00:37 +02:00
parent ac821917d1
commit 5da174aeb0
10 changed files with 232 additions and 42 deletions
--- a/cmd/kubernetesDeploy.go
+++ b/cmd/kubernetesDeploy.go
@@ -15,6 +15,7 @@ import (
 	"text/template"

 	"github.com/SAP/jenkins-library/pkg/docker"
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/SAP/jenkins-library/pkg/kubernetes"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
@@ -38,7 +39,20 @@ func runKubernetesDeploy(config kubernetesDeployOptions, telemetryData *telemetr
 	telemetryData.Custom1 = config.DeployTool

 	if config.DeployTool == "helm" || config.DeployTool == "helm3" {
-		return runHelmDeploy(config, utils, stdout)
+		err := runHelmDeploy(config, utils, stdout)
+		// download and execute teardown script
+		if len(config.TeardownScript) > 0 {
+			log.Entry().Debugf("start running teardownScript script %v", config.TeardownScript)
+			if scriptErr := downloadAndExecuteExtensionScript(config.TeardownScript, config.GithubToken, utils); scriptErr != nil {
+				if err != nil {
+					err = fmt.Errorf("failed to download/run teardownScript script: %v: %w", fmt.Sprint(scriptErr), err)
+				} else {
+					err = scriptErr
+				}
+			}
+			log.Entry().Debugf("finished running teardownScript script %v", config.TeardownScript)
+		}
+		return err
 	} else if config.DeployTool == "kubectl" {
 		return runKubectlDeploy(config, utils, stdout)
 	}
@@ -52,6 +66,16 @@ func runHelmDeploy(config kubernetesDeployOptions, utils kubernetes.DeployUtils,
 	if len(config.DeploymentName) <= 0 {
 		return fmt.Errorf("deployment name has not been set, please configure deploymentName parameter")
 	}
+
+	// download and execute setup script
+	if len(config.SetupScript) > 0 {
+		log.Entry().Debugf("start running setup script %v", config.SetupScript)
+		if err := downloadAndExecuteExtensionScript(config.SetupScript, config.GithubToken, utils); err != nil {
+			return fmt.Errorf("failed to download/run setup setup script: %w", err)
+		}
+		log.Entry().Debugf("finished running setup script %v", config.SetupScript)
+	}
+
 	_, containerRegistry, err := splitRegistryURL(config.ContainerRegistryURL)
 	if err != nil {
 		log.Entry().WithError(err).Fatalf("Container registry url '%v' incorrect", config.ContainerRegistryURL)
@@ -187,6 +211,15 @@ func runHelmDeploy(config kubernetesDeployOptions, utils kubernetes.DeployUtils,
 		log.Entry().WithError(err).Fatal("Helm upgrade call failed")
 	}

+	// download and execute verification script
+	if len(config.VerificationScript) > 0 {
+		log.Entry().Debugf("start running verification script %v", config.VerificationScript)
+		if err := downloadAndExecuteExtensionScript(config.VerificationScript, config.GithubToken, utils); err != nil {
+			return fmt.Errorf("failed to download/run verification script: %w", err)
+		}
+		log.Entry().Debugf("finished running verification script %v", config.VerificationScript)
+	}
+
 	testParams := []string{
 		"test",
 		config.DeploymentName,
@@ -532,3 +565,15 @@ func defineDeploymentValues(config kubernetesDeployOptions, containerRegistry st

 	return dv, nil
 }
+
+func downloadAndExecuteExtensionScript(script, githubToken string, utils kubernetes.DeployUtils) error {
+	setupScript, err := piperhttp.DownloadExecutable(githubToken, utils, utils, script)
+	if err != nil {
+		return fmt.Errorf("failed to download script %v: %w", script, err)
+	}
+	err = utils.RunExecutable(setupScript)
+	if err != nil {
+		return fmt.Errorf("failed to execute script %v: %w", script, err)
+	}
+	return nil
+}
--- a/cmd/kubernetesDeploy_generated.go
+++ b/cmd/kubernetesDeploy_generated.go
@@ -33,6 +33,7 @@ type kubernetesDeployOptions struct {
 	HelmDeployWaitSeconds      int                    `json:"helmDeployWaitSeconds,omitempty"`
 	HelmValues                 []string               `json:"helmValues,omitempty"`
 	ValuesMapping              map[string]interface{} `json:"valuesMapping,omitempty"`
+	GithubToken                string                 `json:"githubToken,omitempty"`
 	Image                      string                 `json:"image,omitempty"`
 	ImageNames                 []string               `json:"imageNames,omitempty"`
 	ImageNameTags              []string               `json:"imageNameTags,omitempty"`
@@ -48,6 +49,9 @@ type kubernetesDeployOptions struct {
 	TillerNamespace            string                 `json:"tillerNamespace,omitempty"`
 	DockerConfigJSON           string                 `json:"dockerConfigJSON,omitempty"`
 	DeployCommand              string                 `json:"deployCommand,omitempty" validate:"possible-values=apply replace"`
+	SetupScript                string                 `json:"setupScript,omitempty"`
+	VerificationScript         string                 `json:"verificationScript,omitempty"`
+	TeardownScript             string                 `json:"teardownScript,omitempty"`
 }

 // KubernetesDeployCommand Deployment to Kubernetes test or production namespace within the specified Kubernetes cluster.
@@ -100,6 +104,7 @@ helm upgrade <deploymentName> <chartPath> --install --force --namespace <namespa
 			}
 			log.RegisterSecret(stepConfig.ContainerRegistryPassword)
 			log.RegisterSecret(stepConfig.ContainerRegistryUser)
+			log.RegisterSecret(stepConfig.GithubToken)
 			log.RegisterSecret(stepConfig.KubeConfig)
 			log.RegisterSecret(stepConfig.KubeToken)
 			log.RegisterSecret(stepConfig.DockerConfigJSON)
@@ -182,6 +187,7 @@ func addKubernetesDeployFlags(cmd *cobra.Command, stepConfig *kubernetesDeployOp
 	cmd.Flags().IntVar(&stepConfig.HelmDeployWaitSeconds, "helmDeployWaitSeconds", 300, "Number of seconds before helm deploy returns.")
 	cmd.Flags().StringSliceVar(&stepConfig.HelmValues, "helmValues", []string{}, "List of helm values as YAML file reference or URL (as per helm parameter description for `-f` / `--values`)")

+	cmd.Flags().StringVar(&stepConfig.GithubToken, "githubToken", os.Getenv("PIPER_githubToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line")
 	cmd.Flags().StringVar(&stepConfig.Image, "image", os.Getenv("PIPER_image"), "Full name of the image to be deployed.")
 	cmd.Flags().StringSliceVar(&stepConfig.ImageNames, "imageNames", []string{}, "List of names of the images to be deployed.")
 	cmd.Flags().StringSliceVar(&stepConfig.ImageNameTags, "imageNameTags", []string{}, "List of full names (registry and tag) of the images to be deployed.")
@@ -197,6 +203,9 @@ func addKubernetesDeployFlags(cmd *cobra.Command, stepConfig *kubernetesDeployOp
 	cmd.Flags().StringVar(&stepConfig.TillerNamespace, "tillerNamespace", os.Getenv("PIPER_tillerNamespace"), "Defines optional tiller namespace for deployments using helm.")
 	cmd.Flags().StringVar(&stepConfig.DockerConfigJSON, "dockerConfigJSON", `.pipeline/docker/config.json`, "Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).")
 	cmd.Flags().StringVar(&stepConfig.DeployCommand, "deployCommand", `apply`, "Only for `deployTool: kubectl`: defines the command `apply` or `replace`. The default is `apply`.")
+	cmd.Flags().StringVar(&stepConfig.SetupScript, "setupScript", os.Getenv("PIPER_setupScript"), "HTTP location of setup script")
+	cmd.Flags().StringVar(&stepConfig.VerificationScript, "verificationScript", os.Getenv("PIPER_verificationScript"), "HTTP location of verification script")
+	cmd.Flags().StringVar(&stepConfig.TeardownScript, "teardownScript", os.Getenv("PIPER_teardownScript"), "HTTP location of teardown script")

 	cmd.MarkFlagRequired("containerRegistryUrl")
 	cmd.MarkFlagRequired("deployTool")
@@ -218,6 +227,7 @@ func kubernetesDeployMetadata() config.StepData {
 					{Name: "kubeTokenCredentialsId", Description: "Jenkins 'Secret text' credentials ID containing token to authenticate to Kubernetes. This is an alternative way to using a kubeconfig file. Details can be found in the [Kubernetes documentation](https://kubernetes.io/docs/reference/access-authn-authz/authentication/).", Type: "jenkins", Aliases: []config.Alias{{Name: "k8sTokenCredentialsId", Deprecated: true}}},
 					{Name: "dockerCredentialsId", Type: "jenkins"},
 					{Name: "dockerConfigJsonCredentialsId", Description: "Jenkins 'Secret file' credentials ID containing Docker config.json (with registry credential(s)).", Type: "jenkins"},
+					{Name: "githubTokenCredentialsId", Description: "Jenkins credentials ID containing the github token.", Type: "jenkins"},
 				},
 				Resources: []config.StepResources{
 					{Name: "deployDescriptor", Type: "stash"},
@@ -417,6 +427,26 @@ func kubernetesDeployMetadata() config.StepData {
 						Mandatory:   false,
 						Aliases:     []config.Alias{},
 					},
+					{
+						Name: "githubToken",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name: "githubTokenCredentialsId",
+								Type: "secret",
+							},
+
+							{
+								Name:    "githubVaultSecretName",
+								Type:    "vaultSecret",
+								Default: "github",
+							},
+						},
+						Scope:     []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{{Name: "access_token"}},
+						Default:   os.Getenv("PIPER_githubToken"),
+					},
 					{
 						Name: "image",
 						ResourceRef: []config.ResourceReference{
@@ -600,6 +630,33 @@ func kubernetesDeployMetadata() config.StepData {
 						Aliases:     []config.Alias{},
 						Default:     `apply`,
 					},
+					{
+						Name:        "setupScript",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_setupScript"),
+					},
+					{
+						Name:        "verificationScript",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_verificationScript"),
+					},
+					{
+						Name:        "teardownScript",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_teardownScript"),
+					},
 				},
 			},
 			Containers: []config.Container{
--- a/cmd/kubernetesDeploy_test.go
+++ b/cmd/kubernetesDeploy_test.go
@@ -1033,6 +1033,35 @@ func TestRunKubernetesDeploy(t *testing.T) {
 		}, mockUtils.Calls[0].Params, "Wrong upgrade parameters")
 	})

+	t.Run("test helm - use extensions", func(t *testing.T) {
+		opts := kubernetesDeployOptions{
+			ContainerRegistryURL:    "https://my.registry:55555",
+			ChartPath:               "path/to/chart",
+			ContainerRegistrySecret: "testSecret",
+			DeploymentName:          "deploymentName",
+			DeployTool:              "helm3",
+			IngressHosts:            []string{},
+			Image:                   "path/to/Image:latest",
+			KubeContext:             "testCluster",
+			Namespace:               "deploymentNamespace",
+			GithubToken:             "testGHToken",
+			SetupScript:             "https://github.com/my/test/setup_script.sh",
+			VerificationScript:      "https://github.com/my/test/verification_script.sh",
+			TeardownScript:          "https://github.com/my/test/teardown_script.sh",
+		}
+		mockUtils := newKubernetesDeployMockUtils()
+		mockUtils.HttpClientMock = &mock.HttpClientMock{HTTPFileUtils: mockUtils.FilesMock}
+
+		var stdout bytes.Buffer
+
+		runKubernetesDeploy(opts, &telemetry.CustomData{}, mockUtils, &stdout)
+
+		assert.Equal(t, 4, len(mockUtils.Calls))
+		assert.Equal(t, ".pipeline/setup_script.sh", mockUtils.Calls[0].Exec)
+		assert.Equal(t, ".pipeline/verification_script.sh", mockUtils.Calls[2].Exec)
+		assert.Equal(t, ".pipeline/teardown_script.sh", mockUtils.Calls[3].Exec)
+	})
+
 	t.Run("test helm v3 - fails without chart path", func(t *testing.T) {
 		opts := kubernetesDeployOptions{
 			ContainerRegistryURL:    "https://my.registry:55555",
--- a/cmd/shellExecute.go
+++ b/cmd/shellExecute.go
@@ -2,9 +2,7 @@ package cmd

 import (
 	"fmt"
-	"net/http"
 	"os/exec"
-	"path/filepath"
 	"strings"

 	"github.com/pkg/errors"
@@ -54,7 +52,7 @@ func runShellExecute(config *shellExecuteOptions, telemetryData *telemetry.Custo
 	for position, source := range config.Sources {

 		if strings.Contains(source, "https") {
-			scriptLocation, err := downloadScript(config, utils, source)
+			scriptLocation, err := piperhttp.DownloadExecutable(config.GithubToken, utils, utils, source)
 			if err != nil {
 				return errors.Wrapf(err, "script download error")
 			}
@@ -105,25 +103,3 @@ func runShellExecute(config *shellExecuteOptions, telemetryData *telemetry.Custo
 func isArgumentAtPosition(scriptArguments []string, index int) bool {
 	return ((len(scriptArguments) > index) && scriptArguments[index] != "")
 }
-
-func downloadScript(config *shellExecuteOptions, utils shellExecuteUtils, url string) (string, error) {
-	header := http.Header{}
-	if len(config.GithubToken) > 0 {
-		header = http.Header{"Authorization": []string{"Token " + config.GithubToken}}
-		header.Set("Accept", "application/vnd.github.v3.raw")
-	}
-
-	log.Entry().Infof("downloading script : %v", url)
-	fileNameParts := strings.Split(url, "/")
-	fileName := fileNameParts[len(fileNameParts)-1]
-	err := utils.DownloadFile(url, filepath.Join(".pipeline", fileName), header, []*http.Cookie{})
-	if err != nil {
-		return "", errors.Wrapf(err, "unable to download script from %v", url)
-	}
-	log.Entry().Infof("downloaded script %v successfully", url)
-	err = fileUtils.Chmod(filepath.Join(".pipeline", fileName), 0555)
-	if err != nil {
-		return "", errors.Wrapf(err, "unable to change script permission for %v", filepath.Join(".pipeline", fileName))
-	}
-	return filepath.Join(".pipeline", fileName), nil
-}
--- a/pkg/http/downloader.go
+++ b/pkg/http/downloader.go
@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"

 	"github.com/pkg/errors"

@@ -55,3 +56,25 @@ func (c *Client) GetRequest(url string, header http.Header, cookies []*http.Cook
 	}
 	return response, nil
 }
+
+// DownloadExecutable downloads a script or another executable and sets appropriate permissions
+func DownloadExecutable(githubToken string, fileUtils piperutils.FileUtils, downloader Downloader, url string) (string, error) {
+	header := http.Header{}
+	if len(githubToken) > 0 {
+		header = http.Header{"Authorization": []string{"Token " + githubToken}}
+		header.Set("Accept", "application/vnd.github.v3.raw")
+	}
+
+	fileNameParts := strings.Split(url, "/")
+	fileName := fileNameParts[len(fileNameParts)-1]
+	fullFileName := filepath.Join(".pipeline", fileName)
+	err := downloader.DownloadFile(url, fullFileName, header, []*http.Cookie{})
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to download script from %v", url)
+	}
+	err = fileUtils.Chmod(fullFileName, 0555)
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to change script permission for %v", fullFileName)
+	}
+	return fullFileName, nil
+}
--- a/pkg/kubernetes/utils.go
+++ b/pkg/kubernetes/utils.go
@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net/http"

 	"github.com/SAP/jenkins-library/pkg/command"
 	piperhttp "github.com/SAP/jenkins-library/pkg/http"
@@ -20,17 +19,17 @@ type DeployUtils interface {
 	Stdout(out io.Writer)
 	Stderr(err io.Writer)
 	RunExecutable(e string, p ...string) error
-	DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error

 	piperutils.FileUtils
 	piperhttp.Uploader
+	piperhttp.Downloader
 }

 // deployUtilsBundle struct  for utils
 type deployUtilsBundle struct {
 	*command.Command
 	*piperutils.Files
-	piperhttp.Uploader
+	*piperhttp.Client
 }

 // NewDeployUtilsBundle initialize using deployUtilsBundle struct
@@ -64,8 +63,8 @@ func NewDeployUtilsBundle(customTLSCertificateLinks []string) DeployUtils {
 				},
 			},
 		},
-		Files:    &piperutils.Files{},
-		Uploader: &httpClient,
+		Files:  &piperutils.Files{},
+		Client: &piperhttp.Client{},
 	}
 	// reroute stderr output to logging framework, stdout will be used for command interactions
 	utils.Stderr(log.Writer())
@@ -97,7 +96,3 @@ func GetChartInfo(chartYamlFile string, utils DeployUtils) (string, string, erro

 	return name, version, nil
 }
-
-func (d *deployUtilsBundle) DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error {
-	return fmt.Errorf("not implemented")
-}
--- a/pkg/mock/httpClient.go
+++ b/pkg/mock/httpClient.go
@@ -14,9 +14,10 @@ import (
 // HttpClientMock mock struct
 type HttpClientMock struct {
 	ClientOptions          []piperhttp.ClientOptions // set by mock
-	FileUploads            map[string]string         // set by mock
-	ReturnFileUploadStatus int                       // expected to be set upfront
-	ReturnFileUploadError  error                     // expected to be set upfront
+	HTTPFileUtils          *FilesMock
+	FileUploads            map[string]string // set by mock
+	ReturnFileUploadStatus int               // expected to be set upfront
+	ReturnFileUploadError  error             // expected to be set upfront
 }

 // SendRequest mock
@@ -52,5 +53,8 @@ func (utils *HttpClientMock) UploadFile(url, file, fieldName string, header http

 // DownloadFile mock
 func (utils *HttpClientMock) DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error {
-	return fmt.Errorf("not implemented")
+	if utils.HTTPFileUtils != nil {
+		utils.HTTPFileUtils.AddFile(filename, []byte("some content"))
+	}
+	return nil
 }
--- a/pkg/mock/httpClient_test.go
+++ b/pkg/mock/httpClient_test.go
@@ -31,14 +31,18 @@ func TestSendRequest(t *testing.T) {
 func TestDownloadFile(t *testing.T) {
 	t.Parallel()
 	t.Run("DownloadFile", func(t *testing.T) {
-		utils := HttpClientMock{}
+		utils := HttpClientMock{
+			HTTPFileUtils: &FilesMock{},
+		}
 		url := "https://localhost"
 		filename := "testFile"
 		var header http.Header
 		var cookies []*http.Cookie
 		err := utils.DownloadFile(url, filename, header, cookies)
-		assert.Error(t, err)
-
+		assert.NoError(t, err)
+		content, err := utils.HTTPFileUtils.FileRead(filename)
+		assert.NoError(t, err)
+		assert.Equal(t, "some content", string(content))
 	})
 }

--- a/resources/metadata/kubernetesDeploy.yaml
+++ b/resources/metadata/kubernetesDeploy.yaml
@@ -43,6 +43,9 @@ spec:
      - name: dockerConfigJsonCredentialsId
        description: Jenkins 'Secret file' credentials ID containing Docker config.json (with registry credential(s)).
        type: jenkins
+      - name: githubTokenCredentialsId
+        description: Jenkins credentials ID containing the github token.
+        type: jenkins
    resources:
      - name: deployDescriptor
        type: stash
@@ -294,6 +297,24 @@ spec:
          - PARAMETERS
          - STAGES
          - STEPS
+      - name: githubToken
+        description: "GitHub personal access token as per
+          https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line"
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: string
+        secret: true
+        aliases:
+          - name: access_token
+        resourceRef:
+          - name: githubTokenCredentialsId
+            type: secret
+          - type: vaultSecret
+            default: github
+            name: githubVaultSecretName
      - name: image
        aliases:
          - name: deployImage
@@ -460,6 +481,41 @@ spec:
        possibleValues:
          - apply
          - replace
+      - name: setupScript
+        type: string
+        description: HTTP location of setup script
+        longDescription: |
+          For helm-based deploymens only!
+          HTTP location of setup script.
+          The script will be downloaded from a GitHub location using the `githubToken` and executed before the installation of the helm package.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+      - name: verificationScript
+        type: string
+        description: HTTP location of verification script
+        longDescription: |
+          For helm-based deploymens only!
+          HTTP location of verification script.
+          The script will be downloaded from a GitHub location using the `githubToken` and executed after installation of the helm package.
+          It can be used to verify if all required artifacts are ready before progressing with for example `helmTest` using the step option `runHelmTests: true`
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+      - name: teardownScript
+        type: string
+        description: HTTP location of teardown script
+        longDescription: |
+          For helm-based deploymens only!
+          HTTP location of setup script.
+          The script will be downloaded from a GitHub location using the `githubToken` and executed at the end of the step.
+          This can for example be used in order to remove a temporary namespace which was created for the test.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
  containers:
    - image: dtzar/helm-kubectl:3.8.1
      workingDir: /config
--- a/vars/kubernetesDeploy.groovy
+++ b/vars/kubernetesDeploy.groovy
@@ -9,6 +9,7 @@ void call(Map parameters = [:]) {
        [type: 'file', id: 'dockerConfigJsonCredentialsId', env: ['PIPER_dockerConfigJSON']],
        [type: 'token', id: 'kubeTokenCredentialsId', env: ['PIPER_kubeToken']],
        [type: 'usernamePassword', id: 'dockerCredentialsId', env: ['PIPER_containerRegistryUser', 'PIPER_containerRegistryPassword']],
+        [type: 'token', id: 'githubTokenCredentialsId', env: ['PIPER_githubToken']],
    ]
    piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials)
 }