fix (gitOpsUpdateDeployment) add CA bundle options to plain clone and commit to trust enterprise github instances (#4602)

* downloading ca cert bundle when added as config

* adding logging statements

* allowing bats test to handle ca cert

* adding info message

* hard coding file names

* including correct http client util bundle

* removing logging message not needed

* adding cert bundle to commit and push

* improving the condition to add ca cert in commit and push

* fixing unit test

* fixing unit test

* fixing unit test

* fixing unit test

* fixing unit test

cmd/batsExecuteTests.go

@@ -108,7 +108,9 @@ func runBatsExecuteTests(config *batsExecuteTestsOptions, telemetryData *telemet
 }
 
 func (b *batsExecuteTestsUtilsBundle) CloneRepo(URL string) error {
-	_, err := pipergit.PlainClone("", "", URL, "bats-core")
+	// ToDo: BatsExecute test needs to check if the repo can come from a
+	// enterprise github instance and needs ca-cert handelling seperately
+	_, err := pipergit.PlainClone("", "", URL, "bats-core", []byte{})
 	return err
 
 }

cmd/gitopsUpdateDeployment.go

@@ -3,9 +3,19 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
 	"github.com/SAP/jenkins-library/pkg/command"
 	"github.com/SAP/jenkins-library/pkg/docker"
 	gitUtil "github.com/SAP/jenkins-library/pkg/git"
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
@@ -13,12 +23,6 @@ import (
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/pkg/errors"
-	"io"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strings"
-	"time"
 )
 
 const toolKubectl = "kubectl"
@@ -27,8 +31,8 @@ const toolKustomize = "kustomize"
 
 type iGitopsUpdateDeploymentGitUtils interface {
 	CommitFiles(filePaths []string, commitMessage, author string) (plumbing.Hash, error)
-	PushChangesToRepository(username, password string, force *bool) error
+	PushChangesToRepository(username, password string, force *bool, caCerts []byte) error
-	PlainClone(username, password, serverURL, directory string) error
+	PlainClone(username, password, serverURL, directory string, caCerts []byte) error
 	ChangeBranch(branchName string) error
 }
 
@@ -36,6 +40,7 @@ type gitopsUpdateDeploymentFileUtils interface {
 	TempDir(dir, pattern string) (name string, err error)
 	RemoveAll(path string) error
 	FileWrite(path string, content []byte, perm os.FileMode) error
+	FileRead(path string) ([]byte, error)
 	Glob(pattern string) ([]string, error)
 }
 
@@ -51,6 +56,25 @@ type gitopsUpdateDeploymentGitUtils struct {
 	repository *git.Repository
 }
 
+type gitopsUpdateDeploymentUtilsBundle struct {
+	*piperhttp.Client
+}
+
+type gitopsUpdateDeploymentUtils interface {
+	DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error
+}
+
+func newGitopsUpdateDeploymentUtilsBundle() gitopsUpdateDeploymentUtils {
+	utils := gitopsUpdateDeploymentUtilsBundle{
+		Client: &piperhttp.Client{},
+	}
+	return &utils
+}
+
+func (g *gitopsUpdateDeploymentUtilsBundle) DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error {
+	return g.Client.DownloadFile(url, filename, header, cookies)
+}
+
 func (g *gitopsUpdateDeploymentGitUtils) CommitFiles(filePaths []string, commitMessage, author string) (plumbing.Hash, error) {
 	for _, path := range filePaths {
 		_, err := g.worktree.Add(path)
@@ -71,13 +95,13 @@ func (g *gitopsUpdateDeploymentGitUtils) CommitFiles(filePaths []string, commitM
 	return commit, nil
 }
 
-func (g *gitopsUpdateDeploymentGitUtils) PushChangesToRepository(username, password string, force *bool) error {
+func (g *gitopsUpdateDeploymentGitUtils) PushChangesToRepository(username, password string, force *bool, caCerts []byte) error {
-	return gitUtil.PushChangesToRepository(username, password, force, g.repository)
+	return gitUtil.PushChangesToRepository(username, password, force, g.repository, caCerts)
 }
 
-func (g *gitopsUpdateDeploymentGitUtils) PlainClone(username, password, serverURL, directory string) error {
+func (g *gitopsUpdateDeploymentGitUtils) PlainClone(username, password, serverURL, directory string, caCerts []byte) error {
 	var err error
-	g.repository, err = gitUtil.PlainClone(username, password, serverURL, directory)
+	g.repository, err = gitUtil.PlainClone(username, password, serverURL, directory, caCerts)
 	if err != nil {
 		return errors.Wrapf(err, "plain clone failed '%s'", serverURL)
 	}
@@ -126,7 +150,12 @@ func runGitopsUpdateDeployment(config *gitopsUpdateDeploymentOptions, command gi
 		}
 	}()
 
-	err = cloneRepositoryAndChangeBranch(config, gitUtils, temporaryFolder)
+	certs, err := downloadCACertbunde(config.CustomTLSCertificateLinks, gitUtils, fileUtils)
+	if err != nil {
+		return err
+	}
+
+	err = cloneRepositoryAndChangeBranch(config, gitUtils, fileUtils, temporaryFolder, certs)
 	if err != nil {
 		return errors.Wrap(err, "repository could not get prepared")
 	}
@@ -190,7 +219,7 @@ func runGitopsUpdateDeployment(config *gitopsUpdateDeploymentOptions, command gi
 		}
 	}
 
-	commit, err := commitAndPushChanges(config, gitUtils, allFiles)
+	commit, err := commitAndPushChanges(config, gitUtils, allFiles, certs)
 	if err != nil {
 		return errors.Wrap(err, "failed to commit and push changes")
 	}
@@ -292,8 +321,9 @@ func logNotRequiredButFilledFieldForKustomize(config *gitopsUpdateDeploymentOpti
 	}
 }
 
-func cloneRepositoryAndChangeBranch(config *gitopsUpdateDeploymentOptions, gitUtils iGitopsUpdateDeploymentGitUtils, temporaryFolder string) error {
+func cloneRepositoryAndChangeBranch(config *gitopsUpdateDeploymentOptions, gitUtils iGitopsUpdateDeploymentGitUtils, fileUtils gitopsUpdateDeploymentFileUtils, temporaryFolder string, certs []byte) error {
-	err := gitUtils.PlainClone(config.Username, config.Password, config.ServerURL, temporaryFolder)
+
+	err := gitUtils.PlainClone(config.Username, config.Password, config.ServerURL, temporaryFolder, certs)
 	if err != nil {
 		return errors.Wrap(err, "failed to plain clone repository")
 	}
@@ -305,6 +335,30 @@ func cloneRepositoryAndChangeBranch(config *gitopsUpdateDeploymentOptions, gitUt
 	return nil
 }
 
+func downloadCACertbunde(customTlsCertificateLinks []string, gitUtils iGitopsUpdateDeploymentGitUtils, fileUtils gitopsUpdateDeploymentFileUtils) ([]byte, error) {
+	certs := []byte{}
+	utils := newGitopsUpdateDeploymentUtilsBundle()
+	if len(customTlsCertificateLinks) > 0 {
+		for _, customTlsCertificateLink := range customTlsCertificateLinks {
+			log.Entry().Infof("Downloading CA certs %s into file '%s'", customTlsCertificateLink, path.Base(customTlsCertificateLink))
+			err := utils.DownloadFile(customTlsCertificateLink, path.Base(customTlsCertificateLink), nil, nil)
+			if err != nil {
+				return certs, nil
+			}
+
+			content, err := fileUtils.FileRead(path.Base(customTlsCertificateLink))
+			if err != nil {
+				return certs, nil
+			}
+			log.Entry().Infof("CA certs added successfully to cert pool")
+
+			certs = append(certs, content...)
+		}
+	}
+
+	return certs, nil
+}
+
 func executeKubectl(config *gitopsUpdateDeploymentOptions, command gitopsUpdateDeploymentExecRunner, filePath string) ([]byte, error) {
 	var outputBytes []byte
 	registryImage, err := buildRegistryPlusImage(config)
@@ -444,7 +498,7 @@ func buildRegistryPlusImageAndTagSeparately(config *gitopsUpdateDeploymentOption
 
 }
 
-func commitAndPushChanges(config *gitopsUpdateDeploymentOptions, gitUtils iGitopsUpdateDeploymentGitUtils, filePaths []string) (plumbing.Hash, error) {
+func commitAndPushChanges(config *gitopsUpdateDeploymentOptions, gitUtils iGitopsUpdateDeploymentGitUtils, filePaths []string, certs []byte) (plumbing.Hash, error) {
 	commitMessage := config.CommitMessage
 
 	if commitMessage == "" {
@@ -456,7 +510,7 @@ func commitAndPushChanges(config *gitopsUpdateDeploymentOptions, gitUtils iGitop
 		return [20]byte{}, errors.Wrap(err, "committing changes failed")
 	}
 
-	err = gitUtils.PushChangesToRepository(config.Username, config.Password, &config.ForcePush)
+	err = gitUtils.PushChangesToRepository(config.Username, config.Password, &config.ForcePush, certs)
 	if err != nil {
 		return [20]byte{}, errors.Wrap(err, "pushing changes failed")
 	}

cmd/gitopsUpdateDeployment_generated.go

@@ -16,20 +16,21 @@ import (
 )
 
 type gitopsUpdateDeploymentOptions struct {
-	BranchName            string   `json:"branchName,omitempty"`
+	BranchName                string   `json:"branchName,omitempty"`
-	CommitMessage         string   `json:"commitMessage,omitempty"`
+	CommitMessage             string   `json:"commitMessage,omitempty"`
-	ServerURL             string   `json:"serverUrl,omitempty"`
+	ServerURL                 string   `json:"serverUrl,omitempty"`
-	ForcePush             bool     `json:"forcePush,omitempty"`
+	ForcePush                 bool     `json:"forcePush,omitempty"`
-	Username              string   `json:"username,omitempty"`
+	Username                  string   `json:"username,omitempty"`
-	Password              string   `json:"password,omitempty"`
+	Password                  string   `json:"password,omitempty"`
-	FilePath              string   `json:"filePath,omitempty"`
+	FilePath                  string   `json:"filePath,omitempty"`
-	ContainerName         string   `json:"containerName,omitempty"`
+	ContainerName             string   `json:"containerName,omitempty"`
-	ContainerRegistryURL  string   `json:"containerRegistryUrl,omitempty"`
+	ContainerRegistryURL      string   `json:"containerRegistryUrl,omitempty"`
-	ContainerImageNameTag string   `json:"containerImageNameTag,omitempty"`
+	ContainerImageNameTag     string   `json:"containerImageNameTag,omitempty"`
-	ChartPath             string   `json:"chartPath,omitempty"`
+	ChartPath                 string   `json:"chartPath,omitempty"`
-	HelmValues            []string `json:"helmValues,omitempty"`
+	HelmValues                []string `json:"helmValues,omitempty"`
-	DeploymentName        string   `json:"deploymentName,omitempty"`
+	DeploymentName            string   `json:"deploymentName,omitempty"`
-	Tool                  string   `json:"tool,omitempty" validate:"possible-values=kubectl helm kustomize"`
+	Tool                      string   `json:"tool,omitempty" validate:"possible-values=kubectl helm kustomize"`
+	CustomTLSCertificateLinks []string `json:"customTlsCertificateLinks,omitempty"`
 }
 
 // GitopsUpdateDeploymentCommand Updates Kubernetes Deployment Manifest in an Infrastructure Git Repository
@@ -155,6 +156,7 @@ func addGitopsUpdateDeploymentFlags(cmd *cobra.Command, stepConfig *gitopsUpdate
 	cmd.Flags().StringSliceVar(&stepConfig.HelmValues, "helmValues", []string{}, "List of helm values as YAML file reference or URL (as per helm parameter description for `-f` / `--values`)")
 	cmd.Flags().StringVar(&stepConfig.DeploymentName, "deploymentName", os.Getenv("PIPER_deploymentName"), "Defines the name of the deployment. In case of `kustomize` this is the name or alias of the image in the `kustomization.yaml`")
 	cmd.Flags().StringVar(&stepConfig.Tool, "tool", `kubectl`, "Defines the tool which should be used to update the deployment description.")
+	cmd.Flags().StringSliceVar(&stepConfig.CustomTLSCertificateLinks, "customTlsCertificateLinks", []string{}, "List containing download links of custom TLS certificates. This is required to ensure trusted connections to registries with custom certificates.")
 
 	cmd.MarkFlagRequired("branchName")
 	cmd.MarkFlagRequired("serverUrl")
@@ -343,6 +345,15 @@ func gitopsUpdateDeploymentMetadata() config.StepData {
 						Aliases:     []config.Alias{},
 						Default:     `kubectl`,
 					},
+					{
+						Name:        "customTlsCertificateLinks",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "[]string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     []string{},
+					},
 				},
 			},
 			Containers: []config.Container{

cmd/gitopsUpdateDeployment_test.go

@@ -772,6 +772,7 @@ type filesMock struct {
 	failOnCreation bool
 	failOnDeletion bool
 	failOnWrite    bool
+	failOnRead     bool
 	failOnGlob     bool
 	path           string
 }
@@ -783,6 +784,13 @@ func (f filesMock) FileWrite(path string, content []byte, perm os.FileMode) erro
 	return piperutils.Files{}.FileWrite(path, content, perm)
 }
 
+func (f filesMock) FileRead(path string) ([]byte, error) {
+	if f.failOnRead {
+		return []byte{}, errors.New("error appeared")
+	}
+	return piperutils.Files{}.FileRead(path)
+}
+
 func (f filesMock) TempDir(dir string, pattern string) (name string, err error) {
 	if f.failOnCreation {
 		return "", errors.New("error appeared")
@@ -848,7 +856,7 @@ func (v *gitUtilsMock) CommitFiles(newFiles []string, commitMessage string, _ st
 	return [20]byte{123}, nil
 }
 
-func (v gitUtilsMock) PushChangesToRepository(_ string, _ string, force *bool) error {
+func (v gitUtilsMock) PushChangesToRepository(_ string, _ string, force *bool, caCerts []byte) error {
 	if v.failOnPush {
 		return errors.New("error on push")
 	}
@@ -858,7 +866,7 @@ func (v gitUtilsMock) PushChangesToRepository(_ string, _ string, force *bool) e
 	return nil
 }
 
-func (v *gitUtilsMock) PlainClone(_, _, _, directory string) error {
+func (v *gitUtilsMock) PlainClone(_, _, _, directory string, caCerts []byte) error {
 	if v.skipClone {
 		return nil
 	}

pkg/git/git.go

@@ -1,13 +1,14 @@
 package git
 
 import (
+	"time"
+
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/pkg/errors"
-	"time"
 )
 
 // utilsWorkTree interface abstraction of git.Worktree to enable tests
@@ -53,14 +54,18 @@ func commitSingleFile(filePath, commitMessage, author string, worktree utilsWork
 }
 
 // PushChangesToRepository Pushes all committed changes in the repository to the remote repository
-func PushChangesToRepository(username, password string, force *bool, repository *git.Repository) error {
+func PushChangesToRepository(username, password string, force *bool, repository *git.Repository, caCerts []byte) error {
-	return pushChangesToRepository(username, password, force, repository)
+	return pushChangesToRepository(username, password, force, repository, caCerts)
 }
 
-func pushChangesToRepository(username, password string, force *bool, repository utilsRepository) error {
+func pushChangesToRepository(username, password string, force *bool, repository utilsRepository, caCerts []byte) error {
 	pushOptions := &git.PushOptions{
 		Auth: &http.BasicAuth{Username: username, Password: password},
 	}
+
+	if len(caCerts) > 0 {
+		pushOptions.CABundle = caCerts
+	}
 	if force != nil {
 		pushOptions.Force = *force
 	}
@@ -72,16 +77,21 @@ func pushChangesToRepository(username, password string, force *bool, repository
 }
 
 // PlainClone Clones a non-bare repository to the provided directory
-func PlainClone(username, password, serverURL, directory string) (*git.Repository, error) {
+func PlainClone(username, password, serverURL, directory string, caCerts []byte) (*git.Repository, error) {
 	abstractedGit := &abstractionGit{}
-	return plainClone(username, password, serverURL, directory, abstractedGit)
+	return plainClone(username, password, serverURL, directory, abstractedGit, caCerts)
 }
 
-func plainClone(username, password, serverURL, directory string, abstractionGit utilsGit) (*git.Repository, error) {
+func plainClone(username, password, serverURL, directory string, abstractionGit utilsGit, caCerts []byte) (*git.Repository, error) {
 	gitCloneOptions := git.CloneOptions{
 		Auth: &http.BasicAuth{Username: username, Password: password},
 		URL:  serverURL,
 	}
+
+	if len(caCerts) > 0 {
+		gitCloneOptions.CABundle = caCerts
+	}
+
 	repository, err := abstractionGit.plainClone(directory, false, &gitCloneOptions)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to clone git")

pkg/git/git_test.go

@@ -51,13 +51,13 @@ func TestPushChangesToRepository(t *testing.T) {
 		t.Parallel()
 		err := pushChangesToRepository("user", "password", nil, RepositoryMock{
 			test: t,
-		})
+		}, []byte{})
 		assert.NoError(t, err)
 	})
 
 	t.Run("error pushing", func(t *testing.T) {
 		t.Parallel()
-		err := pushChangesToRepository("user", "password", nil, RepositoryMockError{})
+		err := pushChangesToRepository("user", "password", nil, RepositoryMockError{}, []byte{})
 		assert.EqualError(t, err, "failed to push commit: error on push commits")
 	})
 }
@@ -67,7 +67,7 @@ func TestPlainClone(t *testing.T) {
 	t.Run("successful clone", func(t *testing.T) {
 		t.Parallel()
 		abstractedGit := &UtilsGitMock{}
-		_, err := plainClone("user", "password", "URL", "directory", abstractedGit)
+		_, err := plainClone("user", "password", "URL", "directory", abstractedGit, []byte{})
 		assert.NoError(t, err)
 		assert.Equal(t, "directory", abstractedGit.path)
 		assert.False(t, abstractedGit.isBare)
@@ -78,7 +78,7 @@ func TestPlainClone(t *testing.T) {
 	t.Run("error on cloning", func(t *testing.T) {
 		t.Parallel()
 		abstractedGit := UtilsGitMockError{}
-		_, err := plainClone("user", "password", "URL", "directory", abstractedGit)
+		_, err := plainClone("user", "password", "URL", "directory", abstractedGit, []byte{})
 		assert.EqualError(t, err, "failed to clone git: error during clone")
 	})
 }

resources/metadata/gitopsUpdateDeployment.yaml

@@ -190,6 +190,13 @@ spec:
           - kubectl
           - helm
           - kustomize
+      - name: customTlsCertificateLinks
+        type: "[]string"
+        description: List containing download links of custom TLS certificates. This is required to ensure trusted connections to registries with custom certificates.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
   containers:
     - image: dtzar/helm-kubectl:3.8.0
       workingDir: /config