sap-jenkins-library/cmd/detectExecuteScan.go

package cmd

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	bd "github.com/SAP/jenkins-library/pkg/blackduck"
	piperhttp "github.com/SAP/jenkins-library/pkg/http"
	"github.com/SAP/jenkins-library/pkg/maven"
	"github.com/pkg/errors"

	"github.com/SAP/jenkins-library/pkg/command"
	"github.com/SAP/jenkins-library/pkg/log"
	"github.com/SAP/jenkins-library/pkg/piperutils"
	"github.com/SAP/jenkins-library/pkg/reporting"
	"github.com/SAP/jenkins-library/pkg/telemetry"
	"github.com/SAP/jenkins-library/pkg/toolrecord"
	"github.com/SAP/jenkins-library/pkg/versioning"
)

type detectUtils interface {
	Abs(path string) (string, error)
	FileExists(filename string) (bool, error)
	FileRemove(filename string) error
	Copy(src, dest string) (int64, error)
	DirExists(dest string) (bool, error)
	FileRead(path string) ([]byte, error)
	FileWrite(path string, content []byte, perm os.FileMode) error
	MkdirAll(path string, perm os.FileMode) error
	Chmod(path string, mode os.FileMode) error
	Glob(pattern string) (matches []string, err error)

	Stdout(out io.Writer)
	Stderr(err io.Writer)
	SetDir(dir string)
	SetEnv(env []string)
	RunExecutable(e string, p ...string) error
	RunShell(shell, script string) error

	DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error
}

type detectUtilsBundle struct {
	*command.Command
	*piperutils.Files
	*piperhttp.Client
}

type blackduckSystem struct {
	Client bd.Client
}

func newDetectUtils() detectUtils {
	utils := detectUtilsBundle{
		Command: &command.Command{
			ErrorCategoryMapping: map[string][]string{
				log.ErrorCompliance.String(): {
					"FAILURE_POLICY_VIOLATION - Detect found policy violations.",
				},
				log.ErrorConfiguration.String(): {
					"FAILURE_CONFIGURATION - Detect was unable to start due to issues with it's configuration.",
				},
			},
		},
		Files:  &piperutils.Files{},
		Client: &piperhttp.Client{},
	}
	utils.Stdout(log.Writer())
	utils.Stderr(log.Writer())
	return &utils
}

func newBlackduckSystem(config detectExecuteScanOptions) *blackduckSystem {
	sys := blackduckSystem{
		Client: bd.NewClient(config.Token, config.ServerURL, &piperhttp.Client{}),
	}
	return &sys
}

func detectExecuteScan(config detectExecuteScanOptions, _ *telemetry.CustomData, influx *detectExecuteScanInflux) {
	influx.step_data.fields.detect = false
	utils := newDetectUtils()
	err := runDetect(config, utils, influx)

	if err != nil {
		log.Entry().
			WithError(err).
			Fatal("failed to execute detect scan")
	}

	influx.step_data.fields.detect = true
	// create Toolrecord file
	toolRecordFileName, err := createToolRecordDetect("./", config)
	if err != nil {
		// do not fail until the framework is well established
		log.Entry().Warning("TR_DETECT: Failed to create toolrecord file "+toolRecordFileName, err)
	}
}

func runDetect(config detectExecuteScanOptions, utils detectUtils, influx *detectExecuteScanInflux) error {
	// detect execution details, see https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/88440888/Sample+Synopsys+Detect+Scan+Configuration+Scenarios+for+Black+Duck
	err := getDetectScript(config, utils)
	if err != nil {
		return fmt.Errorf("failed to download 'detect.sh' script: %w", err)
	}
	defer func() {
		err := utils.FileRemove("detect.sh")
		if err != nil {
			log.Entry().Warnf("failed to delete 'detect.sh' script: %v", err)
		}
	}()
	err = utils.Chmod("detect.sh", 0700)
	if err != nil {
		return err
	}

	if config.InstallArtifacts {
		err := maven.InstallMavenArtifacts(&maven.EvaluateOptions{
			M2Path:              config.M2Path,
			ProjectSettingsFile: config.ProjectSettingsFile,
			GlobalSettingsFile:  config.GlobalSettingsFile,
		}, utils)
		if err != nil {
			return err
		}
	}

	args := []string{"./detect.sh"}
	args, err = addDetectArgs(args, config, utils)
	if err != nil {
		return err
	}
	script := strings.Join(args, " ")

	envs := []string{"BLACKDUCK_SKIP_PHONE_HOME=true"}
	envs = append(envs, config.CustomEnvironmentVariables...)

	utils.SetDir(".")
	utils.SetEnv(envs)

	err = utils.RunShell("/bin/bash", script)
	reportingErr := postScanChecksAndReporting(config, influx, utils, newBlackduckSystem(config))
	if reportingErr != nil {
		log.Entry().Warnf("Failed to generate reports: %v", reportingErr)
	}
	return err
}

func getDetectScript(config detectExecuteScanOptions, utils detectUtils) error {
	if config.ScanOnChanges {
		return utils.DownloadFile("https://raw.githubusercontent.com/blackducksoftware/detect_rescan/master/detect_rescan.sh", "detect.sh", nil, nil)
	}
	return utils.DownloadFile("https://detect.synopsys.com/detect.sh", "detect.sh", nil, nil)
}

func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectUtils) ([]string, error) {
	detectVersionName := getVersionName(config)
	//Split on spaces, the scanPropeties, so that each property is available as a single string
	//instead of all properties being part of a single string
	config.ScanProperties = piperutils.SplitAndTrim(config.ScanProperties, " ")

	if config.ScanOnChanges {
		args = append(args, "--report")
		config.Unmap = false
	}

	if config.Unmap {
		if !piperutils.ContainsString(config.ScanProperties, "--detect.project.codelocation.unmap=true") {
			args = append(args, fmt.Sprintf("--detect.project.codelocation.unmap=true"))
		}
		config.ScanProperties, _ = piperutils.RemoveAll(config.ScanProperties, "--detect.project.codelocation.unmap=false")
	} else {
		//When unmap is set to false, any occurances of unmap=true from scanProperties must be removed
		config.ScanProperties, _ = piperutils.RemoveAll(config.ScanProperties, "--detect.project.codelocation.unmap=true")
	}

	args = append(args, config.ScanProperties...)

	args = append(args, fmt.Sprintf("--blackduck.url=%v", config.ServerURL))
	args = append(args, fmt.Sprintf("--blackduck.api.token=%v", config.Token))
	// ProjectNames, VersionName, GroupName etc can contain spaces and need to be escaped using double quotes in CLI
	// Hence the string need to be surrounded by \"
	args = append(args, fmt.Sprintf("\"--detect.project.name='%v'\"", config.ProjectName))
	args = append(args, fmt.Sprintf("\"--detect.project.version.name='%v'\"", detectVersionName))

	// Groups parameter is added only when there is atleast one non-empty groupname provided
	if len(config.Groups) > 0 && len(config.Groups[0]) > 0 {
		args = append(args, fmt.Sprintf("\"--detect.project.user.groups='%v'\"", strings.Join(config.Groups, ",")))
	}

	// Atleast 1, non-empty category to fail on must be provided
	if len(config.FailOn) > 0 && len(config.FailOn[0]) > 0 {
		args = append(args, fmt.Sprintf("--detect.policy.check.fail.on.severities=%v", strings.Join(config.FailOn, ",")))
	}

	codelocation := config.CodeLocation
	if len(codelocation) == 0 && len(config.ProjectName) > 0 {
		codelocation = fmt.Sprintf("%v/%v", config.ProjectName, detectVersionName)
	}
	args = append(args, fmt.Sprintf("\"--detect.code.location.name='%v'\"", codelocation))

	if len(config.ScanPaths) > 0 && len(config.ScanPaths[0]) > 0 {
		args = append(args, fmt.Sprintf("--detect.blackduck.signature.scanner.paths=%v", strings.Join(config.ScanPaths, ",")))
	}

	if len(config.DependencyPath) > 0 {
		args = append(args, fmt.Sprintf("--detect.source.path=%v", config.DependencyPath))
	} else {
		args = append(args, fmt.Sprintf("--detect.source.path='.'"))
	}

	if len(config.IncludedPackageManagers) > 0 {
		args = append(args, fmt.Sprintf("--detect.included.detector.types=%v", strings.ToUpper(strings.Join(config.IncludedPackageManagers, ","))))
	}

	if len(config.ExcludedPackageManagers) > 0 {
		args = append(args, fmt.Sprintf("--detect.excluded.detector.types=%v", strings.ToUpper(strings.Join(config.ExcludedPackageManagers, ","))))
	}

	if len(config.MavenExcludedScopes) > 0 {
		args = append(args, fmt.Sprintf("--detect.maven.excluded.scopes=%v", strings.ToLower(strings.Join(config.MavenExcludedScopes, ","))))
	}

	if len(config.DetectTools) > 0 {
		args = append(args, fmt.Sprintf("--detect.tools=%v", strings.Join(config.DetectTools, ",")))
	}

	mavenArgs, err := maven.DownloadAndGetMavenParameters(config.GlobalSettingsFile, config.ProjectSettingsFile, utils)
	if err != nil {
		return nil, err
	}

	if len(config.M2Path) > 0 {
		absolutePath, err := utils.Abs(config.M2Path)
		if err != nil {
			return nil, err
		}
		mavenArgs = append(mavenArgs, fmt.Sprintf("-Dmaven.repo.local=%v", absolutePath))
	}

	if len(mavenArgs) > 0 {
		args = append(args, fmt.Sprintf("\"--detect.maven.build.command='%v'\"", strings.Join(mavenArgs, " ")))
	}

	return args, nil
}

func getVersionName(config detectExecuteScanOptions) string {
	detectVersionName := config.CustomScanVersion
	if len(detectVersionName) > 0 {
		log.Entry().Infof("Using custom version: %v", detectVersionName)
	} else {
		detectVersionName = versioning.ApplyVersioningModel(config.VersioningModel, config.Version)
	}
	return detectVersionName
}

func postScanChecksAndReporting(config detectExecuteScanOptions, influx *detectExecuteScanInflux, utils detectUtils, sys *blackduckSystem) error {
	vulns, _, err := getVulnsAndComponents(config, influx, sys)
	if err != nil {
		return err
	}
	scanReport := createVulnerabilityReport(config, vulns, influx, sys)
	paths, err := writeVulnerabilityReports(scanReport, config, utils)

	policyStatus, err := getPolicyStatus(config, influx, sys)
	policyReport := createPolicyStatusReport(config, policyStatus, influx, sys)
	policyReportPaths, err := writePolicyStatusReports(policyReport, config, utils)

	paths = append(paths, policyReportPaths...)
	piperutils.PersistReportsAndLinks("detectExecuteScan", "", paths, nil)
	if err != nil {
		return errors.Wrapf(err, "failed to check and report scan results")
	}
	policyJsonErr := writeIpPolicyJson(config, utils, paths, sys)
	if policyJsonErr != nil {
		return errors.Wrapf(policyJsonErr, "failed to write IP policy violations json file")
	}
	return nil
}

func getVulnsAndComponents(config detectExecuteScanOptions, influx *detectExecuteScanInflux, sys *blackduckSystem) (*bd.Vulnerabilities, *bd.Components, error) {
	detectVersionName := getVersionName(config)
	vulns, err := sys.Client.GetVulnerabilities(config.ProjectName, detectVersionName)
	if err != nil {
		return nil, nil, err
	}

	majorVulns := 0
	activeVulns := 0
	for _, vuln := range vulns.Items {
		if isActiveVulnerability(vuln) {
			activeVulns++
			if isMajorVulnerability(vuln) {
				majorVulns++
			}
		}
	}
	influx.detect_data.fields.vulnerabilities = activeVulns
	influx.detect_data.fields.major_vulnerabilities = majorVulns
	influx.detect_data.fields.minor_vulnerabilities = activeVulns - majorVulns

	components, err := sys.Client.GetComponents(config.ProjectName, detectVersionName)
	if err != nil {
		return vulns, nil, err
	}
	influx.detect_data.fields.components = components.TotalCount

	return vulns, components, nil
}

func createVulnerabilityReport(config detectExecuteScanOptions, vulns *bd.Vulnerabilities, influx *detectExecuteScanInflux, sys *blackduckSystem) reporting.ScanReport {
	versionName := getVersionName(config)
	versionUrl, _ := sys.Client.GetProjectVersionLink(config.ProjectName, versionName)
	scanReport := reporting.ScanReport{
		Title: "BlackDuck Security Vulnerability Report",
		Subheaders: []reporting.Subheader{
			{Description: "BlackDuck Project Name ", Details: config.ProjectName},
			{Description: "BlackDuck Project Version ", Details: fmt.Sprintf("<a href='%v'>%v</a>", versionUrl, versionName)},
		},
		Overview: []reporting.OverviewRow{
			{Description: "Total number of vulnerabilities ", Details: fmt.Sprint(influx.detect_data.fields.vulnerabilities)},
			{Description: "Total number of Critical/High vulnerabilties ", Details: fmt.Sprint(influx.detect_data.fields.major_vulnerabilities)},
		},
		SuccessfulScan: influx.detect_data.fields.major_vulnerabilities == 0,
		ReportTime:     time.Now(),
	}

	detailTable := reporting.ScanDetailTable{
		NoRowsMessage: "No publicly known vulnerabilities detected",
		Headers: []string{
			"Vulnerability Name",
			"Severity",
			"Overall Score",
			"Base Score",
			"Component Name",
			"Component Version",
			"Description",
			"Status",
		},
		WithCounter:   true,
		CounterHeader: "Entry#",
	}

	vulnItems := vulns.Items
	sort.Slice(vulnItems, func(i, j int) bool {
		return vulnItems[i].OverallScore > vulnItems[j].OverallScore
	})

	for _, vuln := range vulnItems {
		row := reporting.ScanRow{}
		row.AddColumn(vuln.VulnerabilityWithRemediation.VulnerabilityName, 0)
		row.AddColumn(vuln.VulnerabilityWithRemediation.Severity, 0)

		var scoreStyle reporting.ColumnStyle = reporting.Yellow
		if isMajorVulnerability(vuln) {
			scoreStyle = reporting.Red
		}
		if !isActiveVulnerability(vuln) {
			scoreStyle = reporting.Grey
		}
		row.AddColumn(vuln.VulnerabilityWithRemediation.OverallScore, scoreStyle)
		row.AddColumn(vuln.VulnerabilityWithRemediation.BaseScore, 0)
		row.AddColumn(vuln.Name, 0)
		row.AddColumn(vuln.Version, 0)
		row.AddColumn(vuln.VulnerabilityWithRemediation.Description, 0)
		row.AddColumn(vuln.VulnerabilityWithRemediation.RemediationStatus, 0)

		detailTable.Rows = append(detailTable.Rows, row)
	}

	scanReport.DetailTable = detailTable
	return scanReport
}

func writeVulnerabilityReports(scanReport reporting.ScanReport, config detectExecuteScanOptions, utils detectUtils) ([]piperutils.Path, error) {
	reportPaths := []piperutils.Path{}

	htmlReport, _ := scanReport.ToHTML()
	htmlReportPath := "piper_detect_vulnerability_report.html"
	if err := utils.FileWrite(htmlReportPath, htmlReport, 0666); err != nil {
		log.SetErrorCategory(log.ErrorConfiguration)
		return reportPaths, errors.Wrapf(err, "failed to write html report")
	}
	reportPaths = append(reportPaths, piperutils.Path{Name: "BlackDuck Vulnerability Report", Target: htmlReportPath})

	jsonReport, _ := scanReport.ToJSON()
	if exists, _ := utils.DirExists(reporting.StepReportDirectory); !exists {
		err := utils.MkdirAll(reporting.StepReportDirectory, 0777)
		if err != nil {
			return reportPaths, errors.Wrap(err, "failed to create reporting directory")
		}
	}
	if err := utils.FileWrite(filepath.Join(reporting.StepReportDirectory, fmt.Sprintf("detectExecuteScan_oss_%v.json", fmt.Sprintf("%v", time.Now()))), jsonReport, 0666); err != nil {
		return reportPaths, errors.Wrapf(err, "failed to write json report")
	}

	return reportPaths, nil
}

func getPolicyStatus(config detectExecuteScanOptions, influx *detectExecuteScanInflux, sys *blackduckSystem) (*bd.PolicyStatus, error) {
	policyStatus, err := sys.Client.GetPolicyStatus(config.ProjectName, getVersionName(config))
	if err != nil {
		return nil, err
	}

	totalViolations := 0
	for _, level := range policyStatus.SeverityLevels {
		totalViolations += level.Value
	}
	influx.detect_data.fields.policy_violations = totalViolations

	return policyStatus, nil
}

func createPolicyStatusReport(config detectExecuteScanOptions, policyStatus *bd.PolicyStatus, influx *detectExecuteScanInflux, sys *blackduckSystem) reporting.ScanReport {
	versionName := getVersionName(config)
	versionUrl, _ := sys.Client.GetProjectVersionLink(config.ProjectName, versionName)
	policyReport := reporting.ScanReport{
		Title: "BlackDuck Policy Violations Report",
		Subheaders: []reporting.Subheader{
			{Description: "BlackDuck project name ", Details: config.ProjectName},
			{Description: "BlackDuck project version name", Details: fmt.Sprintf("<a href='%v'>%v</a>", versionUrl, versionName)},
		},
		Overview: []reporting.OverviewRow{
			{Description: "Overall Policy Violation Status", Details: policyStatus.OverallStatus},
			{Description: "Total Number of Policy Vioaltions", Details: fmt.Sprint(influx.detect_data.fields.policy_violations)},
		},
		SuccessfulScan: influx.detect_data.fields.policy_violations > 0,
		ReportTime:     time.Now(),
	}

	detailTable := reporting.ScanDetailTable{
		Headers: []string{
			"Policy Severity Level", "Number of Components in Violation",
		},
		WithCounter: false,
	}

	for _, level := range policyStatus.SeverityLevels {
		row := reporting.ScanRow{}
		row.AddColumn(level.Name, 0)
		row.AddColumn(level.Value, 0)
		detailTable.Rows = append(detailTable.Rows, row)
	}
	policyReport.DetailTable = detailTable

	return policyReport
}

func writePolicyStatusReports(scanReport reporting.ScanReport, config detectExecuteScanOptions, utils detectUtils) ([]piperutils.Path, error) {
	reportPaths := []piperutils.Path{}

	htmlReport, _ := scanReport.ToHTML()
	htmlReportPath := "piper_detect_policy_violation_report.html"
	if err := utils.FileWrite(htmlReportPath, htmlReport, 0666); err != nil {
		log.SetErrorCategory(log.ErrorConfiguration)
		return reportPaths, errors.Wrapf(err, "failed to write html report")
	}
	reportPaths = append(reportPaths, piperutils.Path{Name: "BlackDuck Policy Violation Report", Target: htmlReportPath})

	jsonReport, _ := scanReport.ToJSON()
	if exists, _ := utils.DirExists(reporting.StepReportDirectory); !exists {
		err := utils.MkdirAll(reporting.StepReportDirectory, 0777)
		if err != nil {
			return reportPaths, errors.Wrap(err, "failed to create reporting directory")
		}
	}
	if err := utils.FileWrite(filepath.Join(reporting.StepReportDirectory, fmt.Sprintf("detectExecuteScan_policy_%v.json", fmt.Sprintf("%v", time.Now()))), jsonReport, 0666); err != nil {
		return reportPaths, errors.Wrapf(err, "failed to write json report")
	}

	return reportPaths, nil
}

func writeIpPolicyJson(config detectExecuteScanOptions, utils detectUtils, paths []piperutils.Path, sys *blackduckSystem) error {
	components, err := sys.Client.GetComponentsWithLicensePolicyRule(config.ProjectName, getVersionName(config))
	if err != nil {
		errors.Wrapf(err, "failed to get License Policy Violations")
		return err
	}

	violationCount := getActivePolicyViolations(components)
	violations := struct {
		PolicyViolations int      `json:"policyViolations"`
		Reports          []string `json:"reports"`
	}{
		PolicyViolations: violationCount,
		Reports:          []string{},
	}

	for _, path := range paths {
		violations.Reports = append(violations.Reports, path.Target)
	}
	if files, err := utils.Glob("**/*BlackDuck_RiskReport.pdf"); err == nil && len(files) > 0 {
		// there should only be one RiskReport thus only taking the first one
		_, reportFile := filepath.Split(files[0])
		violations.Reports = append(violations.Reports, reportFile)
	}

	violationContent, err := json.Marshal(violations)
	if err != nil {
		return fmt.Errorf("failed to marshal policy violation data: %w", err)
	}

	err = utils.FileWrite("blackduck-ip.json", violationContent, 0666)
	if err != nil {
		return fmt.Errorf("failed to write policy violation report: %w", err)
	}
	return nil
}

func getActivePolicyViolations(components *bd.Components) int {
	if components.TotalCount == 0 {
		return 0
	}
	activeViolations := 0
	for _, component := range components.Items {
		if isActivePolicyViolation(component.PolicyStatus) {
			activeViolations++
		}
	}
	return activeViolations
}

func isActivePolicyViolation(status string) bool {
	if status == "IN_VIOLATION" {
		return true
	}
	return false
}

func isActiveVulnerability(v bd.Vulnerability) bool {
	switch v.VulnerabilityWithRemediation.RemediationStatus {
	case "NEW":
		return true
	case "REMEDIATION_REQUIRED":
		return true
	case "NEEDS_REVIEW":
		return true
	default:
		return false
	}
}

func isMajorVulnerability(v bd.Vulnerability) bool {
	switch v.VulnerabilityWithRemediation.Severity {
	case "CRITICAL":
		return true
	case "HIGH":
		return true
	default:
		return false
	}
}

// create toolrecord file for detect
//
//
func createToolRecordDetect(workspace string, config detectExecuteScanOptions) (string, error) {
	record := toolrecord.New(workspace, "detectExecute", config.ServerURL)

	projectId := ""  // todo needs more research; according to synopsis documentation
	productURL := "" // relevant ids can be found in the logfile
	err := record.AddKeyData("project",
		projectId,
		config.ProjectName,
		productURL)
	if err != nil {
		return "", err
	}
	record.AddContext("DetectTools", config.DetectTools)
	err = record.Persist()
	if err != nil {
		return "", err
	}
	return record.GetFileName(), nil
}