sap-jenkins-library/cmd/sonarExecuteScan.go

package cmd

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/bmatcuk/doublestar"
	"github.com/pkg/errors"

	"github.com/SAP/jenkins-library/pkg/command"
	piperhttp "github.com/SAP/jenkins-library/pkg/http"
	keytool "github.com/SAP/jenkins-library/pkg/java"
	"github.com/SAP/jenkins-library/pkg/log"
	"github.com/SAP/jenkins-library/pkg/orchestrator"
	"github.com/SAP/jenkins-library/pkg/piperutils"
	SonarUtils "github.com/SAP/jenkins-library/pkg/sonar"
	"github.com/SAP/jenkins-library/pkg/telemetry"
	"github.com/SAP/jenkins-library/pkg/versioning"
)

type sonarSettings struct {
	workingDir  string
	binary      string
	environment []string
	options     []string
}

func (s *sonarSettings) addEnvironment(element string) {
	s.environment = append(s.environment, element)
}

func (s *sonarSettings) addOption(element string) {
	s.options = append(s.options, element)
}

var (
	sonar sonarSettings

	execLookPath    = exec.LookPath
	fileUtilsExists = piperutils.FileExists
	fileUtilsUnzip  = piperutils.Unzip
	osRename        = os.Rename
	osStat          = os.Stat
	doublestarGlob  = doublestar.Glob
)

const (
	javaBinaries       = "sonar.java.binaries="
	javaLibraries      = "sonar.java.libraries="
	coverageExclusions = "sonar.coverage.exclusions="
	pomXMLPattern      = "**/pom.xml"
)

func sonarExecuteScan(config sonarExecuteScanOptions, _ *telemetry.CustomData, influx *sonarExecuteScanInflux) {
	runner := command.Command{
		ErrorCategoryMapping: map[string][]string{
			log.ErrorConfiguration.String(): {
				"You must define the following mandatory properties for '*': *",
				"org.sonar.java.AnalysisException: Your project contains .java files, please provide compiled classes with sonar.java.binaries property, or exclude them from the analysis with sonar.exclusions property.",
				"ERROR: Invalid value for *",
				"java.lang.IllegalStateException: No files nor directories matching '*'",
			},
			log.ErrorInfrastructure.String(): {
				"ERROR: SonarQube server [*] can not be reached",
				"Caused by: java.net.SocketTimeoutException: timeout",
				"java.lang.IllegalStateException: Fail to request *",
				"java.lang.IllegalStateException: Fail to download plugin [*] into *",
			},
		},
	}
	// reroute command output to logging framework
	runner.Stdout(log.Writer())
	runner.Stderr(log.Writer())
	// client for downloading the sonar-scanner
	downloadClient := &piperhttp.Client{}
	downloadClient.SetOptions(piperhttp.ClientOptions{TransportTimeout: 20 * time.Second})
	// client for talking to the SonarQube API
	apiClient := &piperhttp.Client{}
	//TODO: implement certificate handling
	apiClient.SetOptions(piperhttp.ClientOptions{TransportSkipVerification: true})

	sonar = sonarSettings{
		workingDir:  "./",
		binary:      "sonar-scanner",
		environment: []string{},
		options:     []string{},
	}

	influx.step_data.fields.sonar = false
	fileUtils := piperutils.Files{}
	if err := runSonar(config, downloadClient, &runner, apiClient, &fileUtils, influx); err != nil {
		if log.GetErrorCategory() == log.ErrorUndefined && runner.GetExitCode() == 2 {
			// see https://github.com/SonarSource/sonar-scanner-cli/blob/adb67d645c3bcb9b46f29dea06ba082ebec9ba7a/src/main/java/org/sonarsource/scanner/cli/Exit.java#L25
			log.SetErrorCategory(log.ErrorConfiguration)
		}
		log.Entry().WithError(err).Fatal("Execution failed")
	}
	influx.step_data.fields.sonar = true
}

func runSonar(config sonarExecuteScanOptions, client piperhttp.Downloader, runner command.ExecRunner, apiClient SonarUtils.Sender, utils piperutils.FileUtils, influx *sonarExecuteScanInflux) error {
	// Set config based on orchestrator-specific environment variables
	detectParametersFromCI(&config)

	if len(config.ServerURL) > 0 {
		sonar.addEnvironment("SONAR_HOST_URL=" + config.ServerURL)
	}
	if len(config.Token) == 0 {
		log.Entry().Warn("sonar token not set")
		// use token provided by sonar-scanner-jenkins plugin
		// https://github.com/SonarSource/sonar-scanner-jenkins/blob/441ef2f485884758b60767bed2ef8a1a0a7fc863/src/main/java/hudson/plugins/sonar/SonarBuildWrapper.java#L132
		if len(os.Getenv("SONAR_AUTH_TOKEN")) > 0 {
			log.Entry().Info("using token from env var SONAR_AUTH_TOKEN")
			config.Token = os.Getenv("SONAR_AUTH_TOKEN")
		}
	}
	if len(config.Token) > 0 {
		sonar.addEnvironment("SONAR_TOKEN=" + config.Token)
	}
	if len(config.Organization) > 0 {
		sonar.addOption("sonar.organization=" + config.Organization)
	}
	if len(config.Version) > 0 {
		version := config.CustomScanVersion
		if len(version) > 0 {
			log.Entry().Infof("Using custom version: %v", version)
		} else {
			version = versioning.ApplyVersioningModel(config.VersioningModel, config.Version)
		}
		sonar.addOption("sonar.projectVersion=" + version)
	}
	if GeneralConfig.Verbose {
		sonar.addOption("sonar.verbose=true")
	}
	if len(config.ProjectKey) > 0 {
		sonar.addOption("sonar.projectKey=" + config.ProjectKey)
	}
	if len(config.M2Path) > 0 && config.InferJavaLibraries {
		sonar.addOption(javaLibraries + filepath.Join(config.M2Path, "**"))
	}
	if len(config.CoverageExclusions) > 0 && !isInOptions(config, coverageExclusions) {
		sonar.addOption(coverageExclusions + strings.Join(config.CoverageExclusions, ","))
	}
	if config.InferJavaBinaries && !isInOptions(config, javaBinaries) {
		addJavaBinaries()
	}
	if config.WaitForQualityGate {
		sonar.addOption("sonar.qualitygate.wait=true")
	}
	if err := handlePullRequest(config); err != nil {
		log.SetErrorCategory(log.ErrorConfiguration)
		return err
	}
	if err := loadSonarScanner(config.SonarScannerDownloadURL, client); err != nil {
		log.SetErrorCategory(log.ErrorInfrastructure)
		return err
	}
	if err := loadCertificates(config.CustomTLSCertificateLinks, client, runner); err != nil {
		log.SetErrorCategory(log.ErrorInfrastructure)
		return err
	}

	if len(config.Options) > 0 {
		sonar.options = append(sonar.options, config.Options...)
	}

	sonar.options = piperutils.PrefixIfNeeded(piperutils.Trim(sonar.options), "-D")

	log.Entry().
		WithField("command", sonar.binary).
		WithField("options", sonar.options).
		WithField("environment", sonar.environment).
		Debug("Executing sonar scan command")
	// execute scan
	runner.SetEnv(sonar.environment)
	err := runner.RunExecutable(sonar.binary, sonar.options...)
	if err != nil {
		return err
	}

	// as PRs are handled locally for legacy SonarQube systems, no measurements will be fetched.
	if len(config.ChangeID) > 0 && config.LegacyPRHandling {
		return nil
	}

	// load task results
	taskReport, err := SonarUtils.ReadTaskReport(sonar.workingDir)
	if err != nil {
		log.Entry().WithError(err).Warning("no scan report found")
		return nil
	}
	// write links JSON
	links := []piperutils.Path{
		{
			Target: taskReport.DashboardURL,
			Name:   "Sonar Web UI",
		},
	}
	piperutils.PersistReportsAndLinks("sonarExecuteScan", sonar.workingDir, utils, nil, links)

	if len(config.Token) == 0 {
		log.Entry().Warn("no measurements are fetched due to missing credentials")
		return nil
	}
	taskService := SonarUtils.NewTaskService(taskReport.ServerURL, config.Token, taskReport.TaskID, apiClient)
	// wait for analysis task to complete
	err = taskService.WaitForTask()
	if err != nil {
		return err
	}
	// fetch number of issues by severity
	issueService := SonarUtils.NewIssuesService(taskReport.ServerURL, config.Token, taskReport.ProjectKey, config.Organization, config.BranchName, config.ChangeID, apiClient)
	influx.sonarqube_data.fields.blocker_issues, err = issueService.GetNumberOfBlockerIssues()
	if err != nil {
		return err
	}
	influx.sonarqube_data.fields.critical_issues, err = issueService.GetNumberOfCriticalIssues()
	if err != nil {
		return err
	}
	influx.sonarqube_data.fields.major_issues, err = issueService.GetNumberOfMajorIssues()
	if err != nil {
		return err
	}
	influx.sonarqube_data.fields.minor_issues, err = issueService.GetNumberOfMinorIssues()
	if err != nil {
		return err
	}
	influx.sonarqube_data.fields.info_issues, err = issueService.GetNumberOfInfoIssues()
	if err != nil {
		return err
	}

	reportData := SonarUtils.ReportData{
		ServerURL:    taskReport.ServerURL,
		ProjectKey:   taskReport.ProjectKey,
		TaskID:       taskReport.TaskID,
		ChangeID:     config.ChangeID,
		BranchName:   config.BranchName,
		Organization: config.Organization,
		NumberOfIssues: SonarUtils.Issues{
			Blocker:  influx.sonarqube_data.fields.blocker_issues,
			Critical: influx.sonarqube_data.fields.critical_issues,
			Major:    influx.sonarqube_data.fields.major_issues,
			Minor:    influx.sonarqube_data.fields.minor_issues,
			Info:     influx.sonarqube_data.fields.info_issues,
		}}

	componentService := SonarUtils.NewMeasuresComponentService(taskReport.ServerURL, config.Token, taskReport.ProjectKey, config.Organization, config.BranchName, config.ChangeID, apiClient)
	cov, err := componentService.GetCoverage()
	if err != nil {
		log.Entry().Warnf("failed to retrieve sonar coverage data: %v", err)
	} else {
		reportData.Coverage = cov
	}

	loc, err := componentService.GetLinesOfCode()
	if err != nil {
		log.Entry().Warnf("failed to retrieve sonar lines of code data: %v", err)
	} else {
		reportData.LinesOfCode = loc
	}

	log.Entry().Debugf("Influx values: %v", influx.sonarqube_data.fields)

	err = SonarUtils.WriteReport(reportData, sonar.workingDir, ioutil.WriteFile)

	if err != nil {
		return err
	}
	return nil
}

// isInOptions returns true, if the given property is already provided in config.Options.
func isInOptions(config sonarExecuteScanOptions, property string) bool {
	property = strings.TrimSuffix(property, "=")
	return piperutils.ContainsStringPart(config.Options, property)
}

func addJavaBinaries() {
	pomFiles, err := doublestarGlob(pomXMLPattern)
	if err != nil {
		log.Entry().Warnf("failed to glob for pom modules: %v", err)
		return
	}
	var binaries []string

	var classesDirs = []string{"classes", "test-classes"}

	for _, pomFile := range pomFiles {
		module := filepath.Dir(pomFile)
		for _, classDir := range classesDirs {
			classesPath := filepath.Join(module, "target", classDir)
			_, err := osStat(classesPath)
			if err == nil {
				binaries = append(binaries, classesPath)
			}
		}
	}
	if len(binaries) > 0 {
		sonar.addOption(javaBinaries + strings.Join(binaries, ","))
	}
}

func handlePullRequest(config sonarExecuteScanOptions) error {
	if len(config.ChangeID) > 0 {
		if config.LegacyPRHandling {
			// see https://docs.sonarqube.org/display/PLUG/GitHub+Plugin
			sonar.addOption("sonar.analysis.mode=preview")
			sonar.addOption("sonar.github.pullRequest=" + config.ChangeID)
			if len(config.GithubAPIURL) > 0 {
				sonar.addOption("sonar.github.endpoint=" + config.GithubAPIURL)
			}
			if len(config.GithubToken) > 0 {
				sonar.addOption("sonar.github.oauth=" + config.GithubToken)
			}
			if len(config.Owner) > 0 && len(config.Repository) > 0 {
				sonar.addOption("sonar.github.repository=" + config.Owner + "/" + config.Repository)
			}
			if config.DisableInlineComments {
				sonar.addOption("sonar.github.disableInlineComments=" + strconv.FormatBool(config.DisableInlineComments))
			}
		} else {
			// see https://sonarcloud.io/documentation/analysis/pull-request/
			provider := strings.ToLower(config.PullRequestProvider)
			if provider == "github" {
				if len(config.Owner) > 0 && len(config.Repository) > 0 {
					sonar.addOption("sonar.pullrequest.github.repository=" + config.Owner + "/" + config.Repository)
				}
			} else {
				return errors.New("Pull-Request provider '" + provider + "' is not supported!")
			}
			sonar.addOption("sonar.pullrequest.key=" + config.ChangeID)
			sonar.addOption("sonar.pullrequest.base=" + config.ChangeTarget)
			sonar.addOption("sonar.pullrequest.branch=" + config.ChangeBranch)
			sonar.addOption("sonar.pullrequest.provider=" + provider)
		}
	} else if len(config.BranchName) > 0 {
		sonar.addOption("sonar.branch.name=" + config.BranchName)
	}
	return nil
}

func loadSonarScanner(url string, client piperhttp.Downloader) error {
	if scannerPath, err := execLookPath(sonar.binary); err == nil {
		// using existing sonar-scanner
		log.Entry().WithField("path", scannerPath).Debug("Using local sonar-scanner")
	} else if len(url) != 0 {
		// download sonar-scanner-cli into TEMP folder
		log.Entry().WithField("url", url).Debug("Downloading sonar-scanner")
		tmpFolder := getTempDir()
		defer os.RemoveAll(tmpFolder) // clean up
		archive := filepath.Join(tmpFolder, path.Base(url))
		if err := client.DownloadFile(url, archive, nil, nil); err != nil {
			return errors.Wrap(err, "Download of sonar-scanner failed")
		}
		// unzip sonar-scanner-cli
		log.Entry().WithField("source", archive).WithField("target", tmpFolder).Debug("Extracting sonar-scanner")
		if _, err := fileUtilsUnzip(archive, tmpFolder); err != nil {
			return errors.Wrap(err, "Extraction of sonar-scanner failed")
		}
		// move sonar-scanner-cli to .sonar-scanner/
		toolPath := ".sonar-scanner"
		foldername := strings.ReplaceAll(strings.ReplaceAll(archive, ".zip", ""), "cli-", "")
		log.Entry().WithField("source", foldername).WithField("target", toolPath).Debug("Moving sonar-scanner")
		if err := osRename(foldername, toolPath); err != nil {
			return errors.Wrap(err, "Moving of sonar-scanner failed")
		}
		// update binary path
		sonar.binary = filepath.Join(getWorkingDir(), toolPath, "bin", sonar.binary)
		log.Entry().Debug("Download completed")
	}
	return nil
}

func loadCertificates(certificateList []string, client piperhttp.Downloader, runner command.ExecRunner) error {
	truststorePath := filepath.Join(getWorkingDir(), ".certificates")
	truststoreFile := filepath.Join(truststorePath, "cacerts")

	if exists, _ := fileUtilsExists(truststoreFile); exists {
		// use local existing trust store
		sonar.addEnvironment("SONAR_SCANNER_OPTS=" + keytool.GetMavenOpts(truststoreFile))
		log.Entry().WithField("trust store", truststoreFile).Info("Using local trust store")
	} else if len(certificateList) > 0 {
		// create download temp dir
		tmpFolder := getTempDir()
		defer os.RemoveAll(tmpFolder) // clean up
		if err := os.MkdirAll(truststorePath, 0777); err != nil {
			log.Entry().Warningf("failed to create directory %v: %v", truststorePath, err)
		}
		// copying existing truststore
		defaultTruststorePath := keytool.GetDefaultTruststorePath()
		if exists, _ := fileUtilsExists(defaultTruststorePath); exists {
			if err := keytool.ImportTruststore(runner, truststoreFile, defaultTruststorePath); err != nil {
				return errors.Wrap(err, "Copying existing keystore failed")
			}
		}
		// use local created trust store with downloaded certificates
		for _, certificate := range certificateList {
			target := filepath.Join(tmpFolder, path.Base(certificate))
			log.Entry().WithField("source", certificate).WithField("target", target).Info("Downloading TLS certificate")
			// download certificate
			if err := client.DownloadFile(certificate, target, nil, nil); err != nil {
				return errors.Wrapf(err, "Download of TLS certificate failed")
			}
			// add certificate to keystore
			if err := keytool.ImportCert(runner, truststoreFile, target); err != nil {
				log.Entry().Warnf("Adding certificate to keystore failed")
				// return errors.Wrap(err, "Adding certificate to keystore failed")
			}
		}
		sonar.addEnvironment("SONAR_SCANNER_OPTS=" + keytool.GetMavenOpts(truststoreFile))
		log.Entry().WithField("trust store", truststoreFile).Info("Using local trust store")
	} else {
		log.Entry().Debug("Download of TLS certificates skipped")
	}
	return nil
}

func getWorkingDir() string {
	workingDir, err := os.Getwd()
	if err != nil {
		log.Entry().WithError(err).WithField("path", workingDir).Debug("Retrieving of work directory failed")
	}
	return workingDir
}

func getTempDir() string {
	tmpFolder, err := ioutil.TempDir(".", "temp-")
	if err != nil {
		log.Entry().WithError(err).WithField("path", tmpFolder).Debug("Creating temp directory failed")
	}
	return tmpFolder
}

// Fetches parameters from environment variables and updates the options accordingly (only if not already set)
func detectParametersFromCI(options *sonarExecuteScanOptions) {
	provider, err := orchestrator.NewOrchestratorSpecificConfigProvider()
	if err != nil {
		log.Entry().WithError(err).Warning("Cannot infer config from CI environment")
		return
	}

	if provider.IsPullRequest() {
		config := provider.GetPullRequestConfig()
		if len(options.ChangeBranch) == 0 {
			log.Entry().Info("Inferring parameter changeBranch from environment: " + config.Branch)
			options.ChangeBranch = config.Branch
		}
		if len(options.ChangeTarget) == 0 {
			log.Entry().Info("Inferring parameter changeTarget from environment: " + config.Base)
			options.ChangeTarget = config.Base
		}
		if len(options.ChangeID) == 0 {
			log.Entry().Info("Inferring parameter changeId from environment: " + config.Key)
			options.ChangeID = config.Key
		}
	} else {
		branch := provider.GetBranch()
		if options.InferBranchName && len(options.BranchName) == 0 {
			log.Entry().Info("Inferring parameter branchName from environment: " + branch)
			options.BranchName = branch
		}
	}
}