added logic of fetching golang private packages for whitesource step (#4595)

* added logic of fetching golang private packages for whitesource step and detectExecuteScan step

* changed logic of checking by config.PrivateModulesGitToken

* moved func prepareGolangPrivatePackages to golangBuild.go

* fix (gitOpsUpdateDeployment) add CA bundle options to plain clone and commit to trust enterprise github instances (#4602)

* downloading ca cert bundle when added as config

* adding logging statements

* allowing bats test to handle ca cert

* adding info message

* hard coding file names

* including correct http client util bundle

* removing logging message not needed

* adding cert bundle to commit and push

* improving the condition to add ca cert in commit and push

* fixing unit test

* fixing unit test

* fixing unit test

* fixing unit test

* fixing unit test

* feat(kanikoExecute): add dockerfilePath param to multipleImages (#4569)

* add containerDockerfilePath param to multipleImages

* rename ContainerDockerfilePath param to DockerfilePath

* Fix trailing spaces

---------

Co-authored-by: Egor Balakin <egor.balakin@sap.com>
Co-authored-by: Vyacheslav Starostin <32613074+vstarostin@users.noreply.github.com>

* fix(helm): forward sourceRepositoryCredentialsId from groovy to go layer (#4604)

forward sourceRepositoryCredentialsId from groovy to go layer in the same way how this is done for the targetRepositoryCredentialsId

* feat(config): exporting generateConfig function and applying minor changes (#4605)

* exporting generateConfig function and applying minor changes

* Added setConfigOptions to set configOptions variable.
Added possibility to set format output, json or yaml for now.

* Correcting mistake on cmd/getDefaults.go

Co-authored-by: Jordi van Liempt <35920075+jliempt@users.noreply.github.com>

---------

Co-authored-by: Jordi van Liempt <35920075+jliempt@users.noreply.github.com>

* moved func prepareGolangPrivatePackages to pkg/golang

---------

Co-authored-by: Akramdzhon Azamov <MY_NAME@example.com>
Co-authored-by: Andrei Kireev <andrei.kireev@sap.com>
Co-authored-by: Anil Keshav <anil.keshav@sap.com>
Co-authored-by: Egor Balakin <14162703+m1ron0xFF@users.noreply.github.com>
Co-authored-by: Egor Balakin <egor.balakin@sap.com>
Co-authored-by: Vyacheslav Starostin <32613074+vstarostin@users.noreply.github.com>
Co-authored-by: Marcus Holl <marcus.holl@sap.com>
Co-authored-by: Jk1484 <35270240+Jk1484@users.noreply.github.com>
Co-authored-by: Jordi van Liempt <35920075+jliempt@users.noreply.github.com>

cmd/detectExecuteScan.go

@@ -15,6 +15,7 @@ import (
 	bd "github.com/SAP/jenkins-library/pkg/blackduck"
 	"github.com/SAP/jenkins-library/pkg/command"
 	piperGithub "github.com/SAP/jenkins-library/pkg/github"
+	"github.com/SAP/jenkins-library/pkg/golang"
 	piperhttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/maven"
@@ -138,6 +139,14 @@ func detectExecuteScan(config detectExecuteScanOptions, _ *telemetry.CustomData,
 	if err != nil {
 		log.Entry().WithError(err).Warning("Failed to get GitHub client")
 	}
+
+	if config.PrivateModules == "" && config.PrivateModulesGitToken != "" {
+		//configuring go private packages
+		if err := golang.PrepareGolangPrivatePackages("detectExecuteStep", config.PrivateModules, config.PrivateModulesGitToken); err != nil {
+			log.Entry().Warningf("couldn't set private packages for golang, error: %s", err.Error())
+		}
+	}
+
 	utils := newDetectUtils(client)
 	if err := runDetect(ctx, config, utils, influx); err != nil {
 		log.Entry().

cmd/detectExecuteScan_generated.go

@@ -62,6 +62,8 @@ type detectExecuteScanOptions struct {
 	ExcludedDirectories         []string `json:"excludedDirectories,omitempty"`
 	NpmDependencyTypesExcluded  []string `json:"npmDependencyTypesExcluded,omitempty" validate:"possible-values=NONE DEV PEER"`
 	NpmArguments                []string `json:"npmArguments,omitempty"`
+	PrivateModules              string   `json:"privateModules,omitempty"`
+	PrivateModulesGitToken      string   `json:"privateModulesGitToken,omitempty"`
 }
 
 type detectExecuteScanInflux struct {
@@ -195,6 +197,7 @@ Please configure your BlackDuck server Url using the serverUrl parameter and the
 			}
 			log.RegisterSecret(stepConfig.Token)
 			log.RegisterSecret(stepConfig.GithubToken)
+			log.RegisterSecret(stepConfig.PrivateModulesGitToken)
 
 			if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 {
 				sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID)
@@ -305,6 +308,8 @@ func addDetectExecuteScanFlags(cmd *cobra.Command, stepConfig *detectExecuteScan
 	cmd.Flags().StringSliceVar(&stepConfig.ExcludedDirectories, "excludedDirectories", []string{}, "List of directories which should be excluded from the scan.")
 	cmd.Flags().StringSliceVar(&stepConfig.NpmDependencyTypesExcluded, "npmDependencyTypesExcluded", []string{}, "List of npm dependency types which Detect should exclude from the BOM.")
 	cmd.Flags().StringSliceVar(&stepConfig.NpmArguments, "npmArguments", []string{}, "List of additional arguments that Detect will add at then end of the npm ls command line when Detect executes the NPM CLI Detector on an NPM project.")
+	cmd.Flags().StringVar(&stepConfig.PrivateModules, "privateModules", os.Getenv("PIPER_privateModules"), "Tells go which modules shall be considered to be private (by setting [GOPRIVATE](https://pkg.go.dev/cmd/go#hdr-Configuration_for_downloading_non_public_code)).")
+	cmd.Flags().StringVar(&stepConfig.PrivateModulesGitToken, "privateModulesGitToken", os.Getenv("PIPER_privateModulesGitToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.")
 
 	cmd.MarkFlagRequired("token")
 	cmd.MarkFlagRequired("projectName")
@@ -324,6 +329,7 @@ func detectExecuteScanMetadata() config.StepData {
 				Secrets: []config.StepSecrets{
 					{Name: "detectTokenCredentialsId", Description: "Jenkins 'Secret text' credentials ID containing the API token used to authenticate with the Synopsis Detect (formerly BlackDuck) Server.", Type: "jenkins", Aliases: []config.Alias{{Name: "apiTokenCredentialsId", Deprecated: false}}},
 					{Name: "githubTokenCredentialsId", Description: "Jenkins 'Secret text' credentials ID containing token to authenticate to GitHub.", Type: "jenkins"},
+					{Name: "golangPrivateModulesGitTokenCredentialsId", Description: "Jenkins 'Username with password' credentials ID containing username/password for http access to your git repos where your go private modules are stored.", Type: "jenkins"},
 				},
 				Resources: []config.StepResources{
 					{Name: "buildDescriptor", Type: "stash"},
@@ -737,6 +743,36 @@ func detectExecuteScanMetadata() config.StepData {
 						Aliases:     []config.Alias{{Name: "detect/npmArguments"}},
 						Default:     []string{},
 					},
+					{
+						Name:        "privateModules",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_privateModules"),
+					},
+					{
+						Name: "privateModulesGitToken",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name:  "golangPrivateModulesGitTokenCredentialsId",
+								Param: "password",
+								Type:  "secret",
+							},
+
+							{
+								Name:    "golangPrivateModulesGitTokenVaultSecret",
+								Type:    "vaultSecret",
+								Default: "golang",
+							},
+						},
+						Scope:     []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{},
+						Default:   os.Getenv("PIPER_privateModulesGitToken"),
+					},
 				},
 			},
 			Containers: []config.Container{

cmd/golangBuild_generated.go

@@ -498,7 +498,7 @@ func golangBuildMetadata() config.StepData {
 					{
 						Name:        "privateModules",
 						ResourceRef: []config.ResourceReference{},
-						Scope:       []string{"STEPS", "STAGES", "PARAMETERS"},
+						Scope:       []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"},
 						Type:        "string",
 						Mandatory:   false,
 						Aliases:     []config.Alias{},

cmd/whitesourceExecuteScan.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/SAP/jenkins-library/pkg/command"
 	"github.com/SAP/jenkins-library/pkg/format"
+	"github.com/SAP/jenkins-library/pkg/golang"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/npm"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
@@ -157,6 +158,13 @@ func whitesourceExecuteScan(config ScanOptions, _ *telemetry.CustomData, commonP
 }
 
 func runWhitesourceExecuteScan(ctx context.Context, config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment, influx *whitesourceExecuteScanInflux) error {
+	if config != nil && config.PrivateModules != "" && config.PrivateModulesGitToken != "" {
+		//configuring go private packages
+		if err := golang.PrepareGolangPrivatePackages("WhitesourceExecuteStep", config.PrivateModules, config.PrivateModulesGitToken); err != nil {
+			log.Entry().Warningf("couldn't set private packages for golang, error: %s", err.Error())
+		}
+	}
+
 	if err := resolveAggregateProjectName(config, scan, sys); err != nil {
 		return errors.Wrapf(err, "failed to resolve and aggregate project name")
 	}

cmd/whitesourceExecuteScan_generated.go

@@ -74,6 +74,8 @@ type whitesourceExecuteScanOptions struct {
 	Repository                           string   `json:"repository,omitempty"`
 	Assignees                            []string `json:"assignees,omitempty"`
 	CustomTLSCertificateLinks            []string `json:"customTlsCertificateLinks,omitempty"`
+	PrivateModules                       string   `json:"privateModules,omitempty"`
+	PrivateModulesGitToken               string   `json:"privateModulesGitToken,omitempty"`
 }
 
 type whitesourceExecuteScanCommonPipelineEnvironment struct {
@@ -243,6 +245,7 @@ The step uses the so-called Mend Unified Agent. For details please refer to the
 			log.RegisterSecret(stepConfig.OrgToken)
 			log.RegisterSecret(stepConfig.UserToken)
 			log.RegisterSecret(stepConfig.GithubToken)
+			log.RegisterSecret(stepConfig.PrivateModulesGitToken)
 
 			if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 {
 				sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID)
@@ -366,6 +369,8 @@ func addWhitesourceExecuteScanFlags(cmd *cobra.Command, stepConfig *whitesourceE
 	cmd.Flags().StringVar(&stepConfig.Repository, "repository", os.Getenv("PIPER_repository"), "Set the GitHub repository.")
 	cmd.Flags().StringSliceVar(&stepConfig.Assignees, "assignees", []string{``}, "Defines the assignees for the Github Issue created/updated with the results of the scan as a list of login names.")
 	cmd.Flags().StringSliceVar(&stepConfig.CustomTLSCertificateLinks, "customTlsCertificateLinks", []string{}, "List of download links to custom TLS certificates. This is required to ensure trusted connections to instances with repositories (like nexus) when publish flag is set to true.")
+	cmd.Flags().StringVar(&stepConfig.PrivateModules, "privateModules", os.Getenv("PIPER_privateModules"), "Tells go which modules shall be considered to be private (by setting [GOPRIVATE](https://pkg.go.dev/cmd/go#hdr-Configuration_for_downloading_non_public_code)).")
+	cmd.Flags().StringVar(&stepConfig.PrivateModulesGitToken, "privateModulesGitToken", os.Getenv("PIPER_privateModulesGitToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.")
 
 	cmd.MarkFlagRequired("buildTool")
 	cmd.MarkFlagRequired("orgToken")
@@ -387,6 +392,7 @@ func whitesourceExecuteScanMetadata() config.StepData {
 					{Name: "orgAdminUserTokenCredentialsId", Description: "Jenkins 'Secret text' credentials ID containing Whitesource org admin token.", Type: "jenkins", Aliases: []config.Alias{{Name: "whitesourceOrgAdminUserTokenCredentialsId", Deprecated: false}, {Name: "whitesource/orgAdminUserTokenCredentialsId", Deprecated: true}}},
 					{Name: "dockerConfigJsonCredentialsId", Description: "Jenkins 'Secret file' credentials ID containing Docker config.json (with registry credential(s)). You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).", Type: "jenkins", Aliases: []config.Alias{{Name: "dockerCredentialsId", Deprecated: true}}},
 					{Name: "githubTokenCredentialsId", Description: "Jenkins 'Secret text' credentials ID containing token to authenticate to GitHub.", Type: "jenkins"},
+					{Name: "golangPrivateModulesGitTokenCredentialsId", Description: "Jenkins 'Username with password' credentials ID containing username/password for http access to your git repos where your go private modules are stored.", Type: "jenkins"},
 				},
 				Resources: []config.StepResources{
 					{Name: "buildDescriptor", Type: "stash"},
@@ -967,6 +973,36 @@ func whitesourceExecuteScanMetadata() config.StepData {
 						Aliases:     []config.Alias{},
 						Default:     []string{},
 					},
+					{
+						Name:        "privateModules",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_privateModules"),
+					},
+					{
+						Name: "privateModulesGitToken",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name:  "golangPrivateModulesGitTokenCredentialsId",
+								Param: "password",
+								Type:  "secret",
+							},
+
+							{
+								Name:    "golangPrivateModulesGitTokenVaultSecret",
+								Type:    "vaultSecret",
+								Default: "golang",
+							},
+						},
+						Scope:     []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{},
+						Default:   os.Getenv("PIPER_privateModulesGitToken"),
+					},
 				},
 			},
 			Containers: []config.Container{

pkg/golang/golang.go

@@ -0,0 +1,43 @@
+package golang
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/SAP/jenkins-library/pkg/command"
+)
+
+type utilsBundle struct {
+	command.Command
+}
+
+// prepare golang private packages for whitesource and blackduck(detectExecuteScan)
+func PrepareGolangPrivatePackages(stepName, privateModules, privateModulesGitToken string) error {
+	utils := &utilsBundle{
+		Command: command.Command{
+			StepName: stepName,
+		},
+	}
+	os.Setenv("GOPRIVATE", privateModules)
+	err := gitConfigurationForPrivateModules(privateModules, privateModulesGitToken, utils)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func gitConfigurationForPrivateModules(privateMod string, token string, utils *utilsBundle) error {
+	privateMod = strings.ReplaceAll(privateMod, "/*", "")
+	privateMod = strings.ReplaceAll(privateMod, "*.", "")
+	modules := strings.Split(privateMod, ",")
+	for _, v := range modules {
+		authenticatedRepoURL := fmt.Sprintf("https://%s@%s", token, v)
+		repoBaseURL := fmt.Sprintf("https://%s", v)
+		err := utils.RunExecutable("git", "config", "--global", fmt.Sprintf("url.%s.insteadOf", authenticatedRepoURL), repoBaseURL)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

resources/metadata/detectExecuteScan.yaml

@@ -21,6 +21,9 @@ spec:
       - name: githubTokenCredentialsId
         description: Jenkins 'Secret text' credentials ID containing token to authenticate to GitHub.
         type: jenkins
+      - name: golangPrivateModulesGitTokenCredentialsId
+        description: Jenkins 'Username with password' credentials ID containing username/password for http access to your git repos where your go private modules are stored.
+        type: jenkins
     params:
       - name: token
         aliases:
@@ -489,6 +492,32 @@ spec:
           - PARAMETERS
           - STAGES
           - STEPS
+      - name: privateModules
+        type: "string"
+        description: Tells go which modules shall be considered to be private (by setting [GOPRIVATE](https://pkg.go.dev/cmd/go#hdr-Configuration_for_downloading_non_public_code)).
+        scope:
+          - GENERAL
+          - STEPS
+          - STAGES
+          - PARAMETERS
+        alias:
+          - goprivate
+      - name: privateModulesGitToken
+        description: GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: string
+        secret: true
+        resourceRef:
+          - name: golangPrivateModulesGitTokenCredentialsId
+            type: secret
+            param: password
+          - type: vaultSecret
+            name: golangPrivateModulesGitTokenVaultSecret
+            default: golang
   outputs:
     resources:
       - name: influx

resources/metadata/golangBuild.yaml

@@ -209,6 +209,7 @@ spec:
         type: "string"
         description: Tells go which modules shall be considered to be private (by setting [GOPRIVATE](https://pkg.go.dev/cmd/go#hdr-Configuration_for_downloading_non_public_code)).
         scope:
+          - GENERAL
           - STEPS
           - STAGES
           - PARAMETERS

resources/metadata/whitesourceExecuteScan.yaml

@@ -38,6 +38,9 @@ spec:
       - name: githubTokenCredentialsId
         description: Jenkins 'Secret text' credentials ID containing token to authenticate to GitHub.
         type: jenkins
+      - name: golangPrivateModulesGitTokenCredentialsId
+        description: Jenkins 'Username with password' credentials ID containing username/password for http access to your git repos where your go private modules are stored.
+        type: jenkins
     params:
       - name: agentDownloadUrl
         type: string
@@ -597,6 +600,32 @@ spec:
           - PARAMETERS
           - STAGES
           - STEPS
+      - name: privateModules
+        type: "string"
+        description: Tells go which modules shall be considered to be private (by setting [GOPRIVATE](https://pkg.go.dev/cmd/go#hdr-Configuration_for_downloading_non_public_code)).
+        scope:
+          - GENERAL
+          - STEPS
+          - STAGES
+          - PARAMETERS
+        alias:
+          - goprivate
+      - name: privateModulesGitToken
+        description: GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: string
+        secret: true
+        resourceRef:
+          - name: golangPrivateModulesGitTokenCredentialsId
+            type: secret
+            param: password
+          - type: vaultSecret
+            name: golangPrivateModulesGitTokenVaultSecret
+            default: golang
     resources:
       - name: buildDescriptor
         type: stash

vars/detectExecuteScan.groovy

@@ -12,7 +12,8 @@ void call(Map parameters = [:]) {
     parameters = DownloadCacheUtils.injectDownloadCacheInParameters(script, parameters, BuildTool.MAVEN)
     List credentials = [
         [type: 'token', id: 'detectTokenCredentialsId', env: ['PIPER_token']],
-        [type: 'token', id: 'githubTokenCredentialsId', env: ['PIPER_githubToken']]
+        [type: 'token', id: 'githubTokenCredentialsId', env: ['PIPER_githubToken']],
+        [type: 'usernamePassword', id: 'golangPrivateModulesGitTokenCredentialsId', env: ['PIPER_privateModulesGitUsername', 'PIPER_privateModulesGitToken']]
     ]
     piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials)
 }

vars/whitesourceExecuteScan.groovy

@@ -18,6 +18,7 @@ void call(Map parameters = [:]) {
         [type: 'token', id: 'userTokenCredentialsId', env: ['PIPER_userToken']],
         [type: 'token', id: 'githubTokenCredentialsId', env: ['PIPER_githubToken']],
         [type: 'file', id: 'dockerConfigJsonCredentialsId', env: ['PIPER_dockerConfigJSON']],
+        [type: 'usernamePassword', id: 'golangPrivateModulesGitTokenCredentialsId', env: ['PIPER_privateModulesGitUsername', 'PIPER_privateModulesGitToken']]
     ]
     piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials)
 }