feat(golangBuild): support private modules (#3471)

* feat(golangBuild): support private module repositories

cmd/golangBuild.go

@@ -5,14 +5,21 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"regexp"
 	"strings"
 	"text/template"
 
+	"github.com/SAP/jenkins-library/pkg/certutils"
 	"github.com/SAP/jenkins-library/pkg/command"
+	"github.com/SAP/jenkins-library/pkg/goget"
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/piperenv"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
+
+	"golang.org/x/mod/modfile"
+	"golang.org/x/mod/module"
 )
 
 const (
@@ -25,10 +32,10 @@ const (
 
 type golangBuildUtils interface {
 	command.ExecRunner
+	goget.Client
 
-	FileExists(filename string) (bool, error)
+	piperutils.FileUtils
-	FileRead(path string) ([]byte, error)
+	piperhttp.Sender
-	FileWrite(path string, content []byte, perm os.FileMode) error
 
 	// Add more methods here, or embed additional interfaces, or remove/replace as required.
 	// The golangBuildUtils interface should be descriptive of your runtime dependencies,
@@ -39,6 +46,8 @@ type golangBuildUtils interface {
 type golangBuildUtilsBundle struct {
 	*command.Command
 	*piperutils.Files
+	piperhttp.Sender
+	goget.Client
 
 	// Embed more structs as necessary to implement methods or interfaces you add to golangBuildUtils.
 	// Structs embedded in this way must each have a unique set of methods attached.
@@ -46,10 +55,24 @@ type golangBuildUtilsBundle struct {
 	// golangBuildUtilsBundle and forward to the implementation of the dependency.
 }
 
-func newGolangBuildUtils() golangBuildUtils {
+func newGolangBuildUtils(config golangBuildOptions) golangBuildUtils {
+	httpClientOptions := piperhttp.ClientOptions{}
+
+	if len(config.CustomTLSCertificateLinks) > 0 {
+		httpClientOptions.TransportSkipVerification = false
+		httpClientOptions.TrustedCerts = config.CustomTLSCertificateLinks
+	}
+
+	httpClient := piperhttp.Client{}
+	httpClient.SetOptions(httpClientOptions)
+
 	utils := golangBuildUtilsBundle{
 		Command: &command.Command{},
 		Files:   &piperutils.Files{},
+		Sender:  &httpClient,
+		Client: &goget.ClientImpl{
+			HTTPClient: &httpClient,
+		},
 	}
 	// Reroute command output to logging framework
 	utils.Stdout(log.Writer())
@@ -60,7 +83,7 @@ func newGolangBuildUtils() golangBuildUtils {
 func golangBuild(config golangBuildOptions, telemetryData *telemetry.CustomData) {
 	// Utils can be used wherever the command.ExecRunner interface is expected.
 	// It can also be used for example as a mavenExecRunner.
-	utils := newGolangBuildUtils()
+	utils := newGolangBuildUtils(config)
 
 	// Error situations will be bubbled up until they reach the line below which will then stop execution
 	// through the log.Entry().Fatal() call leading to an os.Exit(1) in the end.
@@ -71,6 +94,10 @@ func golangBuild(config golangBuildOptions, telemetryData *telemetry.CustomData)
 }
 
 func runGolangBuild(config *golangBuildOptions, telemetryData *telemetry.CustomData, utils golangBuildUtils) error {
+	err := prepareGolangEnvironment(config, utils)
+	if err != nil {
+		return err
+	}
 
 	// install test pre-requisites only in case testing should be performed
 	if config.RunTests || config.RunIntegrationTests {
@@ -129,6 +156,44 @@ func runGolangBuild(config *golangBuildOptions, telemetryData *telemetry.CustomD
 	return nil
 }
 
+func prepareGolangEnvironment(config *golangBuildOptions, utils golangBuildUtils) error {
+	// configure truststore
+	err := certutils.CertificateUpdate(config.CustomTLSCertificateLinks, utils, utils, "/etc/ssl/certs/ca-certificates.crt") // TODO reimplement
+
+	if config.PrivateModules == "" {
+		return nil
+	}
+
+	if config.PrivateModulesGitToken == "" {
+		return fmt.Errorf("please specify a token for fetching private git modules")
+	}
+
+	// pass private repos to go process
+	os.Setenv("GOPRIVATE", config.PrivateModules)
+
+	repoURLs, err := lookupGolangPrivateModulesRepositories(config.PrivateModules, utils)
+
+	if err != nil {
+		return err
+	}
+
+	// configure credentials git shall use for pulling repos
+	for _, repoURL := range repoURLs {
+		if match, _ := regexp.MatchString("(?i)^https?://", repoURL); !match {
+			continue
+		}
+
+		authenticatedRepoURL := strings.Replace(repoURL, "://", fmt.Sprintf("://%s@", config.PrivateModulesGitToken), 1)
+
+		err = utils.RunExecutable("git", "config", "--global", fmt.Sprintf("url.%s.insteadOf", authenticatedRepoURL), fmt.Sprintf("%s", repoURL))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func runGolangTests(config *golangBuildOptions, utils golangBuildUtils) (bool, error) {
 	// execute gotestsum in order to have more output options
 	if err := utils.RunExecutable("gotestsum", "--junitfile", golangUnitTestOutput, "--", fmt.Sprintf("-coverprofile=%v", coverageFile), "./..."); err != nil {
@@ -266,3 +331,45 @@ func splitTargetArchitecture(architecture string) (string, string) {
 	architectureParts := strings.Split(architecture, ",")
 	return architectureParts[0], architectureParts[1]
 }
+
+// lookupPrivateModulesRepositories returns a slice of all modules that match the given glob pattern
+func lookupGolangPrivateModulesRepositories(globPattern string, utils golangBuildUtils) ([]string, error) {
+	if globPattern == "" {
+		return []string{}, nil
+	}
+
+	if modFileExists, err := utils.FileExists("go.mod"); err != nil {
+		return nil, err
+	} else if !modFileExists {
+		return []string{}, nil // nothing to do
+	}
+
+	modFileContent, err := utils.FileRead("go.mod")
+	if err != nil {
+		return nil, err
+	}
+
+	goModFile, err := modfile.Parse("go.mod", modFileContent, nil)
+	if err != nil {
+		return nil, err
+	} else if goModFile.Require == nil {
+		return []string{}, nil // no modules referenced, nothing to do
+	}
+
+	privateModules := []string{}
+
+	for _, goModule := range goModFile.Require {
+		if !module.MatchPrefixPatterns(globPattern, goModule.Mod.Path) {
+			continue
+		}
+
+		repo, err := utils.GetRepositoryURL(goModule.Mod.Path)
+
+		if err != nil {
+			return nil, err
+		}
+
+		privateModules = append(privateModules, repo)
+	}
+	return privateModules, nil
+}

cmd/golangBuild_generated.go

@@ -32,6 +32,8 @@ type golangBuildOptions struct {
 	TargetArchitectures          []string `json:"targetArchitectures,omitempty"`
 	TestOptions                  []string `json:"testOptions,omitempty"`
 	TestResultFormat             string   `json:"testResultFormat,omitempty" validate:"possible-values=junit standard"`
+	PrivateModules               string   `json:"privateModules,omitempty"`
+	PrivateModulesGitToken       string   `json:"privateModulesGitToken,omitempty"`
 }
 
 // GolangBuildCommand This step will execute a golang build.
@@ -70,6 +72,7 @@ If the build is successful the resulting artifact can be uploaded to e.g. a bina
 				log.SetErrorCategory(log.ErrorConfiguration)
 				return err
 			}
+			log.RegisterSecret(stepConfig.PrivateModulesGitToken)
 
 			if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 {
 				sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID)
@@ -144,6 +147,8 @@ func addGolangBuildFlags(cmd *cobra.Command, stepConfig *golangBuildOptions) {
 	cmd.Flags().StringSliceVar(&stepConfig.TargetArchitectures, "targetArchitectures", []string{`linux,amd64`}, "Defines the target architectures for which the build should run using OS and architecture separated by a comma.")
 	cmd.Flags().StringSliceVar(&stepConfig.TestOptions, "testOptions", []string{}, "Options to pass to test as per `go test` documentation (comprises e.g. flags, packages).")
 	cmd.Flags().StringVar(&stepConfig.TestResultFormat, "testResultFormat", `junit`, "Defines the output format of the test results.")
+	cmd.Flags().StringVar(&stepConfig.PrivateModules, "privateModules", os.Getenv("PIPER_privateModules"), "Tells go which modules shall be considered to be private (by setting [GOPRIVATE](https://pkg.go.dev/cmd/go#hdr-Configuration_for_downloading_non_public_code)).")
+	cmd.Flags().StringVar(&stepConfig.PrivateModulesGitToken, "privateModulesGitToken", os.Getenv("PIPER_privateModulesGitToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.")
 
 	cmd.MarkFlagRequired("targetArchitectures")
 }
@@ -158,6 +163,9 @@ func golangBuildMetadata() config.StepData {
 		},
 		Spec: config.StepSpec{
 			Inputs: config.StepInputs{
+				Secrets: []config.StepSecrets{
+					{Name: "golangPrivateModulesGitTokenCredentialsId", Description: "Jenkins 'Username with password' credentials ID containing username/password for http access to your git repos where your go private modules are stored.", Type: "jenkins"},
+				},
 				Parameters: []config.StepParameters{
 					{
 						Name:        "buildFlags",
@@ -303,6 +311,35 @@ func golangBuildMetadata() config.StepData {
 						Aliases:     []config.Alias{},
 						Default:     `junit`,
 					},
+					{
+						Name:        "privateModules",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"STEPS", "STAGES", "PARAMETERS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_privateModules"),
+					},
+					{
+						Name: "privateModulesGitToken",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name: "golangPrivateModulesGitTokenCredentialsId",
+								Type: "secret",
+							},
+
+							{
+								Name:    "golangPrivateModulesGitTokenVaultSecret",
+								Type:    "vaultSecret",
+								Default: "golang",
+							},
+						},
+						Scope:     []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{},
+						Default:   os.Getenv("PIPER_privateModulesGitToken"),
+					},
 				},
 			},
 			Containers: []config.Container{

cmd/golangBuild_test.go

@@ -2,11 +2,14 @@ package cmd
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
 
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/SAP/jenkins-library/pkg/mock"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
 	"github.com/stretchr/testify/assert"
@@ -17,6 +20,18 @@ type golangBuildMockUtils struct {
 	*mock.FilesMock
 }
 
+func (utils golangBuildMockUtils) GetRepositoryURL(module string) (string, error) {
+	return fmt.Sprintf("https://%s.git", module), nil
+}
+
+func (utils golangBuildMockUtils) SendRequest(method string, url string, r io.Reader, header http.Header, cookies []*http.Cookie) (*http.Response, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (utils golangBuildMockUtils) SetOptions(options piperhttp.ClientOptions) {
+	// not implemented
+}
+
 func newGolangBuildTestsUtils() golangBuildMockUtils {
 	utils := golangBuildMockUtils{
 		ExecMockRunner: &mock.ExecMockRunner{},
@@ -432,3 +447,157 @@ func TestRunGolangBuildPerArchitecture(t *testing.T) {
 	})
 
 }
+
+func TestPrepareGolangEnvironment(t *testing.T) {
+	modTestFile := `
+module private.example.com/m
+
+require (
+        example.com/public/module v1.0.0
+        private1.example.com/private/repo v0.1.0
+        private2.example.com/another/repo v0.2.0
+)
+
+go 1.17`
+
+	type expectations struct {
+		envVars          []string
+		commandsExecuted [][]string
+	}
+	tests := []struct {
+		name           string
+		modFileContent string
+		globPattern    string
+		gitToken       string
+		expect         expectations
+	}{
+		{
+			name:           "success - does nothing if privateModules is not set",
+			modFileContent: modTestFile,
+			globPattern:    "",
+			gitToken:       "secret",
+			expect:         expectations{},
+		},
+		{
+			name:           "success - goprivate is set and authentication properly configured",
+			modFileContent: modTestFile,
+			globPattern:    "*.example.com",
+			gitToken:       "secret",
+			expect: expectations{
+				envVars: []string{"GOPRIVATE=*.example.com"},
+				commandsExecuted: [][]string{
+					[]string{"git", "config", "--global", "url.https://secret@private1.example.com/private/repo.git.insteadOf", "https://private1.example.com/private/repo.git"},
+					[]string{"git", "config", "--global", "url.https://secret@private2.example.com/another/repo.git.insteadOf", "https://private2.example.com/another/repo.git"},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			utils := newGolangBuildTestsUtils()
+			utils.FilesMock.AddFile("go.mod", []byte(tt.modFileContent))
+
+			config := golangBuildOptions{}
+			config.PrivateModules = tt.globPattern
+			config.PrivateModulesGitToken = tt.gitToken
+
+			err := prepareGolangEnvironment(&config, &utils)
+
+			if assert.NoError(t, err) {
+				assert.Subset(t, os.Environ(), tt.expect.envVars)
+				assert.Equal(t, len(tt.expect.commandsExecuted), len(utils.Calls))
+
+				for i, expectedCommand := range tt.expect.commandsExecuted {
+					assert.Equal(t, expectedCommand[0], utils.Calls[i].Exec)
+					assert.Equal(t, expectedCommand[1:], utils.Calls[i].Params)
+				}
+			}
+		})
+	}
+}
+
+func TestLookupGolangPrivateModulesRepositories(t *testing.T) {
+	t.Parallel()
+
+	modTestFile := `
+module private.example.com/m
+
+require (
+	example.com/public/module v1.0.0
+	private1.example.com/private/repo v0.1.0
+	private2.example.com/another/repo v0.2.0
+)
+
+go 1.17`
+
+	type expectations struct {
+		repos        []string
+		errorMessage string
+	}
+	tests := []struct {
+		name           string
+		modFileContent string
+		globPattern    string
+		expect         expectations
+	}{
+		{
+			name:           "Does nothing if glob pattern is empty",
+			modFileContent: modTestFile,
+			expect: expectations{
+				repos: []string{},
+			},
+		},
+		{
+			name:           "Does nothing if there is no go.mod file",
+			globPattern:    "private.example.com",
+			modFileContent: "",
+			expect: expectations{
+				repos: []string{},
+			},
+		},
+		{
+			name:           "Detects all private repos using a glob pattern",
+			modFileContent: modTestFile,
+			globPattern:    "*.example.com",
+			expect: expectations{
+				repos: []string{"https://private1.example.com/private/repo.git", "https://private2.example.com/another/repo.git"},
+			},
+		},
+		{
+			name:           "Detects all private repos",
+			modFileContent: modTestFile,
+			globPattern:    "private1.example.com,private2.example.com",
+			expect: expectations{
+				repos: []string{"https://private1.example.com/private/repo.git", "https://private2.example.com/another/repo.git"},
+			},
+		},
+		{
+			name:           "Detects a dedicated repo",
+			modFileContent: modTestFile,
+			globPattern:    "private2.example.com",
+			expect: expectations{
+				repos: []string{"https://private2.example.com/another/repo.git"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			utils := newGolangBuildTestsUtils()
+
+			if tt.modFileContent != "" {
+				utils.FilesMock.AddFile("go.mod", []byte(tt.modFileContent))
+			}
+
+			repos, err := lookupGolangPrivateModulesRepositories(tt.globPattern, utils)
+
+			if tt.expect.errorMessage == "" {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expect.repos, repos)
+			} else {
+				assert.EqualError(t, err, tt.expect.errorMessage)
+			}
+		})
+	}
+}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/GoogleContainerTools/container-diff v0.17.0
 	github.com/Jeffail/gabs/v2 v2.6.1
 	github.com/Masterminds/sprig v2.22.0+incompatible
+	github.com/antchfx/htmlquery v1.2.4
 	github.com/bmatcuk/doublestar v1.3.4
 	github.com/bndr/gojenkins v1.1.1-0.20210520222939-90ed82bfdff6
 	github.com/buildpacks/lifecycle v0.13.0

go.sum

@@ -210,6 +210,10 @@ github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494=
+github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc=
+github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
+github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/arrow/go/arrow v0.0.0-20200601151325-b2287a20f230 h1:5ultmol0yeX75oh1hY78uAFn3dupBQ/QUNxERCkiaUQ=
 github.com/apache/arrow/go/arrow v0.0.0-20200601151325-b2287a20f230/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0=
@@ -2058,6 +2062,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=

pkg/certutils/certutils.go

@@ -9,7 +9,14 @@ import (
 	"github.com/pkg/errors"
 )
 
+// CertificateUpdate adds certificates to the given truststore
 func CertificateUpdate(certLinks []string, httpClient piperhttp.Sender, fileUtils piperutils.FileUtils, caCertsFile string) error {
+	// TODO this implementation doesn't work on non-linux machines, is not failsafe and should be implemented differently
+
+	if len(certLinks) == 0 {
+		return nil
+	}
+
 	caCerts, err := fileUtils.FileRead(caCertsFile)
 	if err != nil {
 		return errors.Wrapf(err, "failed to load file '%v'", caCertsFile)

pkg/goget/goget.go

@@ -0,0 +1,54 @@
+package goget
+
+import (
+	"fmt"
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"net/http"
+	"strings"
+
+	"github.com/antchfx/htmlquery"
+)
+
+// Client .
+type Client interface {
+	GetRepositoryURL(module string) (string, error)
+}
+
+// ClientImpl .
+type ClientImpl struct {
+	HTTPClient piperhttp.Sender
+}
+
+// GetRepositoryURL resolves the repository URL for the given go module. Only git is supported.
+func (c *ClientImpl) GetRepositoryURL(module string) (string, error) {
+	response, err := c.HTTPClient.SendRequest("GET", fmt.Sprintf("https://%s?go-get=1", module), nil, http.Header{}, nil)
+
+	if err != nil {
+		return "", err
+	} else if response.StatusCode == 404 {
+		return "", fmt.Errorf("module '%s' doesn't exist", module)
+	} else if response.StatusCode != 200 {
+		return "", fmt.Errorf("received unexpected response status code: %d", response.StatusCode)
+	}
+
+	html, err := htmlquery.Parse(response.Body)
+
+	if err != nil {
+		return "", fmt.Errorf("unable to parse content: %q", err)
+	}
+
+	metaNode := htmlquery.FindOne(html, "//meta[@name='go-import']/@content")
+
+	if metaNode == nil {
+		return "", fmt.Errorf("couldn't find go-import statement")
+	}
+
+	goImportStatement := htmlquery.SelectAttr(metaNode, "content")
+	goImport := strings.Split(goImportStatement, " ")
+
+	if len(goImport) != 3 || goImport[1] != "git" {
+		return "", fmt.Errorf("unsupported module: '%s'", module)
+	}
+
+	return goImport[2], nil
+}

pkg/goget/goget_test.go

@@ -0,0 +1,144 @@
+package goget
+
+import (
+	"fmt"
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/stretchr/testify/assert"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"testing"
+)
+
+func TestGoGetClient(t *testing.T) {
+	t.Parallel()
+
+	type expectations struct {
+		registryURL  string
+		errorMessage string
+	}
+	tests := []struct {
+		name          string
+		goModuleURL   string
+		vcs           string
+		registryURL   string
+		returnStatus  int
+		returnContent string
+		expect        expectations
+	}{
+		{
+			name:         "success case",
+			goModuleURL:  "example.com/my/repo",
+			vcs:          "git",
+			registryURL:  "https://git.example.com/my/repo.git",
+			returnStatus: 200,
+			expect: expectations{
+				registryURL: "https://git.example.com/my/repo.git",
+			},
+		},
+		{
+			name:         "error - module doesn't exist",
+			goModuleURL:  "example.com/my/repo",
+			vcs:          "git",
+			returnStatus: 404,
+			expect:       expectations{errorMessage: "module 'example.com/my/repo' doesn't exist"},
+		},
+		{
+			name:         "error - unexpected status code",
+			goModuleURL:  "example.com/my/repo",
+			vcs:          "git",
+			returnStatus: 401,
+			expect:       expectations{errorMessage: "received unexpected response status code: 401"},
+		},
+		{
+			name:          "error - endpoint doesn't implement the go-import protocol",
+			returnStatus:  200,
+			returnContent: "<!DOCTYPE html>\n<html lang=\"en\"><head></head></html>",
+			expect:        expectations{errorMessage: "couldn't find go-import statement"},
+		},
+		{
+			name:         "error - unsupported vcs",
+			returnStatus: 200,
+			goModuleURL:  "example.com/my/repo",
+			vcs:          "svn",
+			registryURL:  "https://svn.example.com/my/repo/trunk",
+			expect:       expectations{errorMessage: "unsupported module: 'example.com/my/repo'"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			returnContent := tt.returnContent
+
+			if returnContent == "" {
+				returnContent = fmt.Sprintf("<!DOCTYPE html>\n<html lang=\"en\"><head><meta name=\"go-import\" content=\"%s %s %s\"></head></html>", tt.goModuleURL, tt.vcs, tt.registryURL)
+			}
+
+			goget := ClientImpl{
+				HTTPClient: &httpMock{StatusCode: tt.returnStatus, ResponseBody: returnContent},
+			}
+
+			repo, err := goget.GetRepositoryURL(tt.goModuleURL)
+
+			if tt.expect.errorMessage != "" {
+				assert.EqualError(t, err, tt.expect.errorMessage)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expect.registryURL, repo)
+			}
+		})
+	}
+}
+
+type httpMock struct {
+	Method       string                  // is set during test execution
+	URL          string                  // is set before test execution
+	ResponseBody string                  // is set before test execution
+	Options      piperhttp.ClientOptions // is set during test
+	StatusCode   int                     // is set during test
+	Body         readCloserMock          // is set during test
+	Header       http.Header             // is set during test
+}
+
+func (c *httpMock) SetOptions(options piperhttp.ClientOptions) {
+	c.Options = options
+}
+
+func (c *httpMock) SendRequest(method string, url string, r io.Reader, header http.Header, cookies []*http.Cookie) (*http.Response, error) {
+	c.Method = method
+	c.URL = url
+	c.Header = header
+
+	if r != nil {
+		_, err := ioutil.ReadAll(r)
+
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	c.Body = readCloserMock{Content: c.ResponseBody}
+	res := http.Response{StatusCode: c.StatusCode, Body: &c.Body}
+
+	return &res, nil
+}
+
+type readCloserMock struct {
+	Content string
+	Closed  bool
+}
+
+func (rc readCloserMock) Read(b []byte) (n int, err error) {
+
+	if len(b) < len(rc.Content) {
+		// in real life we would fill the buffer according to buffer size ...
+		return 0, fmt.Errorf("Buffer size (%d) not sufficient, need: %d", len(b), len(rc.Content))
+	}
+	copy(b, rc.Content)
+	return len(rc.Content), io.EOF
+}
+
+func (rc *readCloserMock) Close() error {
+	rc.Closed = true
+	return nil
+}

resources/metadata/golangBuild.yaml

@@ -10,6 +10,10 @@ metadata:
     If the build is successful the resulting artifact can be uploaded to e.g. a binary repository automatically.
 spec:
   inputs:
+    secrets:
+      - name: golangPrivateModulesGitTokenCredentialsId
+        description: Jenkins 'Username with password' credentials ID containing username/password for http access to your git repos where your go private modules are stored.
+        type: jenkins
     params:
       - name: buildFlags
         type: "[]string"
@@ -145,6 +149,30 @@ spec:
           - STEPS
           - STAGES
           - PARAMETERS
+      - name: privateModules
+        type: "string"
+        description: Tells go which modules shall be considered to be private (by setting [GOPRIVATE](https://pkg.go.dev/cmd/go#hdr-Configuration_for_downloading_non_public_code)).
+        scope:
+          - STEPS
+          - STAGES
+          - PARAMETERS
+        alias:
+          - goprivate
+      - name: privateModulesGitToken
+        description: GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: string
+        secret: true
+        resourceRef:
+          - name: golangPrivateModulesGitTokenCredentialsId
+            type: secret
+          - type: vaultSecret
+            name: golangPrivateModulesGitTokenVaultSecret
+            default: golang
   containers:
     - name: golang
       image: golang:1