sap-jenkins-library/cmd/golangBuild.go

package cmd

import (
	"bytes"
	"fmt"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/SAP/jenkins-library/pkg/buildsettings"
	"github.com/SAP/jenkins-library/pkg/certutils"
	"github.com/SAP/jenkins-library/pkg/command"
	"github.com/SAP/jenkins-library/pkg/goget"
	piperhttp "github.com/SAP/jenkins-library/pkg/http"
	"github.com/SAP/jenkins-library/pkg/log"
	"github.com/SAP/jenkins-library/pkg/piperenv"
	"github.com/SAP/jenkins-library/pkg/piperutils"
	"github.com/SAP/jenkins-library/pkg/telemetry"

	"github.com/SAP/jenkins-library/pkg/multiarch"
	"github.com/SAP/jenkins-library/pkg/versioning"

	"golang.org/x/mod/modfile"
	"golang.org/x/mod/module"
)

const (
	coverageFile                = "cover.out"
	golangUnitTestOutput        = "TEST-go.xml"
	golangIntegrationTestOutput = "TEST-integration.xml"
	golangCoberturaPackage      = "github.com/boumenot/gocover-cobertura@latest"
	golangTestsumPackage        = "gotest.tools/gotestsum@latest"
	golangCycloneDXPackage      = "github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@latest"
	sbomFilename                = "bom-golang.xml"
)

type golangBuildUtils interface {
	command.ExecRunner
	goget.Client

	piperutils.FileUtils
	piperhttp.Uploader

	getDockerImageValue(stepName string) (string, error)
	GetExitCode() int
	DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error
	Untar(src string, dest string, stripComponentLevel int) error

	// Add more methods here, or embed additional interfaces, or remove/replace as required.
	// The golangBuildUtils interface should be descriptive of your runtime dependencies,
	// i.e. include everything you need to be able to mock in tests.
	// Unit tests shall be executable in parallel (not depend on global state), and don't (re-)test dependencies.
}

type golangBuildUtilsBundle struct {
	*command.Command
	*piperutils.Files
	piperhttp.Uploader
	httpClient *piperhttp.Client

	goget.Client

	// Embed more structs as necessary to implement methods or interfaces you add to golangBuildUtils.
	// Structs embedded in this way must each have a unique set of methods attached.
	// If there is no struct which implements the method you need, attach the method to
	// golangBuildUtilsBundle and forward to the implementation of the dependency.
}

func (g *golangBuildUtilsBundle) DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error {
	return g.httpClient.DownloadFile(url, filename, header, cookies)
}

func (g *golangBuildUtilsBundle) getDockerImageValue(stepName string) (string, error) {
	return GetDockerImageValue(stepName)
}

func (g *golangBuildUtilsBundle) Untar(src string, dest string, stripComponentLevel int) error {
	return piperutils.Untar(src, dest, stripComponentLevel)
}

func newGolangBuildUtils(config golangBuildOptions) golangBuildUtils {
	httpClientOptions := piperhttp.ClientOptions{}

	if len(config.CustomTLSCertificateLinks) > 0 {
		httpClientOptions.TransportSkipVerification = false
		httpClientOptions.TrustedCerts = config.CustomTLSCertificateLinks
	}

	httpClient := piperhttp.Client{}
	httpClient.SetOptions(httpClientOptions)

	utils := golangBuildUtilsBundle{
		Command: &command.Command{
			StepName: "golangBuild",
		},
		Files:    &piperutils.Files{},
		Uploader: &httpClient,
		Client: &goget.ClientImpl{
			HTTPClient: &httpClient,
		},
		httpClient: &httpClient,
	}
	// Reroute command output to logging framework
	utils.Stdout(log.Writer())
	utils.Stderr(log.Writer())
	return &utils
}

func golangBuild(config golangBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *golangBuildCommonPipelineEnvironment) {
	// Utils can be used wherever the command.ExecRunner interface is expected.
	// It can also be used for example as a mavenExecRunner.
	utils := newGolangBuildUtils(config)

	// Error situations will be bubbled up until they reach the line below which will then stop execution
	// through the log.Entry().Fatal() call leading to an os.Exit(1) in the end.
	err := runGolangBuild(&config, telemetryData, utils, commonPipelineEnvironment)
	if err != nil {
		log.Entry().WithError(err).Fatal("execution of golang build failed")
	}
}

func runGolangBuild(config *golangBuildOptions, telemetryData *telemetry.CustomData, utils golangBuildUtils, commonPipelineEnvironment *golangBuildCommonPipelineEnvironment) error {
	goModFile, err := readGoModFile(utils) // returns nil if go.mod doesnt exist
	if err != nil {
		return err
	}

	if err = prepareGolangEnvironment(config, goModFile, utils); err != nil {
		return err
	}

	// install test pre-requisites only in case testing should be performed
	if config.RunTests || config.RunIntegrationTests {
		if err := utils.RunExecutable("go", "install", golangTestsumPackage); err != nil {
			return fmt.Errorf("failed to install pre-requisite: %w", err)
		}
	}

	if config.CreateBOM {
		if err := utils.RunExecutable("go", "install", golangCycloneDXPackage); err != nil {
			return fmt.Errorf("failed to install pre-requisite: %w", err)
		}
	}

	failedTests := false

	if config.RunTests {
		success, err := runGolangTests(config, utils)
		if err != nil {
			return err
		}
		failedTests = !success
	}

	if config.RunTests && config.ReportCoverage {
		if err := reportGolangTestCoverage(config, utils); err != nil {
			return err
		}
	}

	if config.RunIntegrationTests {
		success, err := runGolangIntegrationTests(config, utils)
		if err != nil {
			return err
		}
		failedTests = failedTests || !success
	}

	if failedTests {
		log.SetErrorCategory(log.ErrorTest)
		return fmt.Errorf("some tests failed")
	}

	if config.RunLint {
		goPath := os.Getenv("GOPATH")
		golangciLintDir := filepath.Join(goPath, "bin")

		if err := retrieveGolangciLint(utils, golangciLintDir, config.GolangciLintURL); err != nil {
			return err
		}

		// hardcode those for now
		lintSettings := map[string]string{
			"reportStyle":      "checkstyle", // readable by Sonar
			"reportOutputPath": "golangci-lint-report.xml",
			"additionalParams": "",
		}

		if err := runGolangciLint(utils, golangciLintDir, lintSettings); err != nil {
			return err
		}
	}

	if config.CreateBOM {
		if err := runBOMCreation(utils, sbomFilename); err != nil {
			return err
		}
	}

	ldflags := ""

	if len(config.LdflagsTemplate) > 0 {
		ldf, err := prepareLdflags(config, utils, GeneralConfig.EnvRootPath)
		if err != nil {
			return err
		}
		ldflags = (*ldf).String()
		log.Entry().Infof("ldflags from template: '%v'", ldflags)
	}

	var binaries []string
	platforms, err := multiarch.ParsePlatformStrings(config.TargetArchitectures)

	if err != nil {
		return err
	}

	for _, platform := range platforms {
		binaryNames, err := runGolangBuildPerArchitecture(config, goModFile, utils, ldflags, platform)

		if err != nil {
			return err
		}

		if len(binaryNames) > 0 {
			binaries = append(binaries, binaryNames...)
		}
	}

	log.Entry().Debugf("creating build settings information...")
	stepName := "golangBuild"
	dockerImage, err := utils.getDockerImageValue(stepName)
	if err != nil {
		return err
	}

	buildConfig := buildsettings.BuildOptions{
		CreateBOM:         config.CreateBOM,
		Publish:           config.Publish,
		BuildSettingsInfo: config.BuildSettingsInfo,
		DockerImage:       dockerImage,
	}
	buildSettingsInfo, err := buildsettings.CreateBuildSettingsInfo(&buildConfig, stepName)
	if err != nil {
		log.Entry().Warnf("failed to create build settings info: %v", err)
	}
	commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo

	if config.Publish {
		if len(config.TargetRepositoryURL) == 0 {
			return fmt.Errorf("there's no target repository for binary publishing configured")
		}

		artifactVersion := config.ArtifactVersion

		if len(artifactVersion) == 0 {
			artifactOpts := versioning.Options{
				VersioningScheme: "library",
			}

			artifact, err := versioning.GetArtifact("golang", "", &artifactOpts, utils)

			if err != nil {
				return err
			}

			artifactVersion, err = artifact.GetVersion()

			if err != nil {
				return err
			}
		}

		if goModFile == nil {
			return fmt.Errorf("go.mod file not found")
		} else if goModFile.Module == nil {
			return fmt.Errorf("go.mod doesn't declare a module path")
		}

		repoClientOptions := piperhttp.ClientOptions{
			Username:     config.TargetRepositoryUser,
			Password:     config.TargetRepositoryPassword,
			TrustedCerts: config.CustomTLSCertificateLinks,
		}

		utils.SetOptions(repoClientOptions)

		var binaryArtifacts piperenv.Artifacts
		for _, binary := range binaries {

			targetPath := fmt.Sprintf("go/%s/%s/%s", goModFile.Module.Mod.Path, artifactVersion, binary)

			separator := "/"

			if strings.HasSuffix(config.TargetRepositoryURL, "/") {
				separator = ""
			}

			targetURL := fmt.Sprintf("%s%s%s", config.TargetRepositoryURL, separator, targetPath)

			log.Entry().Infof("publishing artifact: %s", targetURL)

			response, err := utils.UploadRequest(http.MethodPut, targetURL, binary, "", nil, nil, "binary")

			if err != nil {
				return fmt.Errorf("couldn't upload artifact: %w", err)
			}

			if !(response.StatusCode == 200 || response.StatusCode == 201) {
				return fmt.Errorf("couldn't upload artifact, received status code %d", response.StatusCode)
			}

			binaryArtifacts = append(binaryArtifacts, piperenv.Artifact{
				Name: binary,
			})
		}
		commonPipelineEnvironment.custom.artifacts = binaryArtifacts

	}

	return nil
}

func prepareGolangEnvironment(config *golangBuildOptions, goModFile *modfile.File, utils golangBuildUtils) error {
	// configure truststore
	err := certutils.CertificateUpdate(config.CustomTLSCertificateLinks, utils, utils, "/etc/ssl/certs/ca-certificates.crt") // TODO reimplement

	if config.PrivateModules == "" {
		return nil
	}

	if config.PrivateModulesGitToken == "" {
		return fmt.Errorf("please specify a token for fetching private git modules")
	}

	// pass private repos to go process
	os.Setenv("GOPRIVATE", config.PrivateModules)

	repoURLs, err := lookupGolangPrivateModulesRepositories(goModFile, config.PrivateModules, utils)

	if err != nil {
		return err
	}

	// configure credentials git shall use for pulling repos
	for _, repoURL := range repoURLs {
		if match, _ := regexp.MatchString("(?i)^https?://", repoURL); !match {
			continue
		}

		authenticatedRepoURL := strings.Replace(repoURL, "://", fmt.Sprintf("://%s@", config.PrivateModulesGitToken), 1)

		err = utils.RunExecutable("git", "config", "--global", fmt.Sprintf("url.%s.insteadOf", authenticatedRepoURL), repoURL)
		if err != nil {
			return err
		}
	}

	return nil
}

func runGolangTests(config *golangBuildOptions, utils golangBuildUtils) (bool, error) {
	// execute gotestsum in order to have more output options
	testOptions := []string{"--junitfile", golangUnitTestOutput, "--", fmt.Sprintf("-coverprofile=%v", coverageFile), "./..."}
	testOptions = append(testOptions, config.TestOptions...)
	if err := utils.RunExecutable("gotestsum", testOptions...); err != nil {
		exists, fileErr := utils.FileExists(golangUnitTestOutput)
		if !exists || fileErr != nil {
			log.SetErrorCategory(log.ErrorBuild)
			return false, fmt.Errorf("running tests failed - junit result missing: %w", err)
		}
		exists, fileErr = utils.FileExists(coverageFile)
		if !exists || fileErr != nil {
			log.SetErrorCategory(log.ErrorBuild)
			return false, fmt.Errorf("running tests failed - coverage output missing: %w", err)
		}
		return false, nil
	}
	return true, nil
}

func runGolangIntegrationTests(config *golangBuildOptions, utils golangBuildUtils) (bool, error) {
	// execute gotestsum in order to have more output options
	// for integration tests coverage data is not meaningful and thus not being created
	if err := utils.RunExecutable("gotestsum", "--junitfile", golangIntegrationTestOutput, "--", "-tags=integration", "./..."); err != nil {
		exists, fileErr := utils.FileExists(golangIntegrationTestOutput)
		if !exists || fileErr != nil {
			log.SetErrorCategory(log.ErrorBuild)
			return false, fmt.Errorf("running tests failed: %w", err)
		}
		return false, nil
	}
	return true, nil
}

func reportGolangTestCoverage(config *golangBuildOptions, utils golangBuildUtils) error {
	if config.CoverageFormat == "cobertura" {
		// execute gocover-cobertura in order to create cobertura report
		// install pre-requisites
		if err := utils.RunExecutable("go", "install", golangCoberturaPackage); err != nil {
			return fmt.Errorf("failed to install pre-requisite: %w", err)
		}

		coverageData, err := utils.FileRead(coverageFile)
		if err != nil {
			return fmt.Errorf("failed to read coverage file %v: %w", coverageFile, err)
		}
		utils.Stdin(bytes.NewBuffer(coverageData))

		coverageOutput := bytes.Buffer{}
		utils.Stdout(&coverageOutput)
		options := []string{}
		if config.ExcludeGeneratedFromCoverage {
			options = append(options, "-ignore-gen-files")
		}
		if err := utils.RunExecutable("gocover-cobertura", options...); err != nil {
			log.SetErrorCategory(log.ErrorTest)
			return fmt.Errorf("failed to convert coverage data to cobertura format: %w", err)
		}
		utils.Stdout(log.Writer())

		err = utils.FileWrite("cobertura-coverage.xml", coverageOutput.Bytes(), 0666)
		if err != nil {
			return fmt.Errorf("failed to create cobertura coverage file: %w", err)
		}
		log.Entry().Info("created file cobertura-coverage.xml")
	} else {
		// currently only cobertura and html format supported, thus using html as fallback
		if err := utils.RunExecutable("go", "tool", "cover", "-html", coverageFile, "-o", "coverage.html"); err != nil {
			return fmt.Errorf("failed to create html coverage file: %w", err)
		}
	}
	return nil
}

func retrieveGolangciLint(utils golangBuildUtils, golangciLintDir, golangciLintURL string) error {
	archiveName := "golangci-lint.tar.gz"
	err := utils.DownloadFile(golangciLintURL, archiveName, nil, nil)
	if err != nil {
		return fmt.Errorf("failed to download golangci-lint: %w", err)
	}

	err = utils.Untar(archiveName, golangciLintDir, 1)
	if err != nil {
		return fmt.Errorf("failed to install golangci-lint: %w", err)
	}

	return nil
}

func runGolangciLint(utils golangBuildUtils, golangciLintDir string, lintSettings map[string]string) error {
	binaryPath := filepath.Join(golangciLintDir, "golangci-lint")

	var outputBuffer bytes.Buffer
	utils.Stdout(&outputBuffer)
	err := utils.RunExecutable(binaryPath, "run", "--out-format", lintSettings["reportStyle"])
	if err != nil && utils.GetExitCode() != 1 {
		return fmt.Errorf("running golangci-lint failed: %w", err)
	}

	log.Entry().Infof("lint report: \n" + outputBuffer.String())
	log.Entry().Infof("writing lint report to %s", lintSettings["reportOutputPath"])
	err = utils.FileWrite(lintSettings["reportOutputPath"], outputBuffer.Bytes(), 0644)
	if err != nil {
		return fmt.Errorf("writing golangci-lint report failed: %w", err)
	}

	if utils.GetExitCode() == 1 {
		return fmt.Errorf("golangci-lint found issues, see report above")
	}

	return nil
}

func prepareLdflags(config *golangBuildOptions, utils golangBuildUtils, envRootPath string) (*bytes.Buffer, error) {
	cpe := piperenv.CPEMap{}
	err := cpe.LoadFromDisk(path.Join(envRootPath, "commonPipelineEnvironment"))
	if err != nil {
		log.Entry().Warning("failed to load values from commonPipelineEnvironment")
	}

	log.Entry().Debugf("ldflagsTemplate in use: %v", config.LdflagsTemplate)
	return cpe.ParseTemplate(config.LdflagsTemplate)
}

func runGolangBuildPerArchitecture(config *golangBuildOptions, goModFile *modfile.File, utils golangBuildUtils, ldflags string, architecture multiarch.Platform) ([]string, error) {
	var binaryNames []string

	envVars := os.Environ()
	envVars = append(envVars, fmt.Sprintf("GOOS=%v", architecture.OS), fmt.Sprintf("GOARCH=%v", architecture.Arch))

	if !config.CgoEnabled {
		envVars = append(envVars, "CGO_ENABLED=0")
	}
	utils.SetEnv(envVars)

	buildOptions := []string{"build", "-trimpath"}

	if len(config.Output) > 0 {
		if len(config.Packages) > 1 {
			binaries, outputDir, err := getOutputBinaries(config.Output, config.Packages, utils, architecture)
			if err != nil {
				log.SetErrorCategory(log.ErrorBuild)
				return nil, fmt.Errorf("failed to calculate output binaries or directory, error: %s", err.Error())
			}
			buildOptions = append(buildOptions, "-o", outputDir)
			binaryNames = append(binaryNames, binaries...)
		} else {
			fileExtension := ""
			if architecture.OS == "windows" {
				fileExtension = ".exe"
			}
			binaryName := fmt.Sprintf("%s-%s.%s%s", strings.TrimRight(config.Output, string(os.PathSeparator)), architecture.OS, architecture.Arch, fileExtension)
			buildOptions = append(buildOptions, "-o", binaryName)
			binaryNames = append(binaryNames, binaryName)
		}
	} else {
		// use default name in case no name is defined via Output
		binaryName := path.Base(goModFile.Module.Mod.Path)
		binaryNames = append(binaryNames, binaryName)
	}
	buildOptions = append(buildOptions, config.BuildFlags...)
	if len(ldflags) > 0 {
		buildOptions = append(buildOptions, "-ldflags", ldflags)
	}
	buildOptions = append(buildOptions, config.Packages...)

	if err := utils.RunExecutable("go", buildOptions...); err != nil {
		log.Entry().Debugf("buildOptions: %v", buildOptions)
		log.SetErrorCategory(log.ErrorBuild)
		return nil, fmt.Errorf("failed to run build for %v.%v: %w", architecture.OS, architecture.Arch, err)
	}

	return binaryNames, nil
}

// lookupPrivateModulesRepositories returns a slice of all modules that match the given glob pattern
func lookupGolangPrivateModulesRepositories(goModFile *modfile.File, globPattern string, utils golangBuildUtils) ([]string, error) {
	if globPattern == "" {
		return []string{}, nil
	}

	if goModFile == nil {
		return nil, fmt.Errorf("couldn't find go.mod file")
	} else if goModFile.Require == nil {
		return []string{}, nil // no modules referenced, nothing to do
	}

	privateModules := []string{}

	for _, goModule := range goModFile.Require {
		if !module.MatchPrefixPatterns(globPattern, goModule.Mod.Path) {
			continue
		}

		repo, err := utils.GetRepositoryURL(goModule.Mod.Path)

		if err != nil {
			return nil, err
		}

		privateModules = append(privateModules, repo)
	}
	return privateModules, nil
}

func runBOMCreation(utils golangBuildUtils, outputFilename string) error {
	if err := utils.RunExecutable("cyclonedx-gomod", "mod", "-licenses", "-test", "-output", outputFilename); err != nil {
		return fmt.Errorf("BOM creation failed: %w", err)
	}
	return nil
}

func readGoModFile(utils golangBuildUtils) (*modfile.File, error) {
	modFilePath := "go.mod"

	if modFileExists, err := utils.FileExists(modFilePath); err != nil {
		return nil, err
	} else if !modFileExists {
		return nil, nil
	}

	modFileContent, err := utils.FileRead(modFilePath)
	if err != nil {
		return nil, err
	}

	return modfile.Parse(modFilePath, modFileContent, nil)
}

func getOutputBinaries(out string, packages []string, utils golangBuildUtils, architecture multiarch.Platform) ([]string, string, error) {
	var binaries []string
	outDir := fmt.Sprintf("%s-%s-%s%c", strings.TrimRight(out, string(os.PathSeparator)), architecture.OS, architecture.Arch, os.PathSeparator)

	for _, pkg := range packages {
		ok, err := isMainPackage(utils, pkg)
		if err != nil {
			return nil, "", err
		}

		if ok {
			fileExt := ""
			if architecture.OS == "windows" {
				fileExt = ".exe"
			}
			binaries = append(binaries, filepath.Join(outDir, filepath.Base(pkg)+fileExt))
		}
	}

	return binaries, outDir, nil
}

func isMainPackage(utils golangBuildUtils, pkg string) (bool, error) {
	outBuffer := bytes.NewBufferString("")
	utils.Stdout(outBuffer)
	utils.Stderr(outBuffer)
	err := utils.RunExecutable("go", "list", "-f", "{{ .Name }}", pkg)
	if err != nil {
		return false, err
	}

	if outBuffer.String() != "main" {
		return false, nil
	}

	return true, nil
}