sap-jenkins-library/cmd/whitesourceExecuteScan.go

package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/360EntSecGroup-Skylar/excelize/v2"
	"github.com/SAP/jenkins-library/pkg/command"
	"github.com/SAP/jenkins-library/pkg/log"
	"github.com/SAP/jenkins-library/pkg/piperutils"
	"github.com/SAP/jenkins-library/pkg/telemetry"
	"github.com/SAP/jenkins-library/pkg/versioning"
	"github.com/SAP/jenkins-library/pkg/whitesource"
)

// just to make the lines less long
type ScanOptions = whitesourceExecuteScanOptions
type System = whitesource.System

func whitesourceExecuteScan(config ScanOptions, telemetry *telemetry.CustomData) {
	// reroute cmd output to logging framework
	c := command.Command{}
	c.Stdout(log.Writer())
	c.Stderr(log.Writer())

	sys := whitesource.NewSystem(config.ServiceURL, config.OrgToken, config.UserToken)
	if err := resolveProjectIdentifiers(&c, sys, &config); err != nil {
		log.Entry().WithError(err).Fatal("step execution failed on resolving project identifiers")
	}

	// Generate a vulnerability report for all projects with version = config.ProjectVersion
	if config.AggregateVersionWideReport {
		if err := aggregateVersionWideLibraries(sys, &config); err != nil {
			log.Entry().WithError(err).Fatal("step execution failed on aggregating version wide libraries")
		}
		if err := aggregateVersionWideVulnerabilities(sys, &config); err != nil {
			log.Entry().WithError(err).Fatal("step execution failed on aggregating version wide vulnerabilities")
		}
	} else {
		if err := runWhitesourceScan(&config, sys, telemetry, &c); err != nil {
			log.Entry().WithError(err).Fatal("step execution failed on executing whitesource scan")
		}
	}
}

func runWhitesourceScan(config *ScanOptions, sys *System, _ *telemetry.CustomData, cmd *command.Command) error {
	// Start the scan
	if err := triggerWhitesourceScan(cmd, config); err != nil {
		return err
	}

	// Scan finished: we need to resolve project token again if the project was just created.
	if err := resolveProjectIdentifiers(cmd, sys, config); err != nil {
		return err
	}

	log.Entry().Info("-----------------------------------------------------")
	log.Entry().Infof("Project name: '%s'", config.ProjectName)
	log.Entry().Infof("Product Version: '%s'", config.ProductVersion)
	log.Entry().Infof("Project Token: %s", config.ProjectToken)
	log.Entry().Info("-----------------------------------------------------")

	if config.Reporting {
		paths, err := downloadReports(config, sys)
		if err != nil {
			return err
		}
		piperutils.PersistReportsAndLinks("whitesourceExecuteScan", "", nil, paths)
	}

	// Check for security vulnerabilities and fail the build if cvssSeverityLimit threshold is crossed
	if config.SecurityVulnerabilities {
		if err := checkSecurityViolations(config, sys); err != nil {
			return err
		}
	}
	return nil
}

func resolveProjectIdentifiers(cmd *command.Command, sys *System, config *ScanOptions) error {
	if config.ProjectName == "" || config.ProductVersion == "" {
		opts := &versioning.Options{}
		artifact, err := versioning.GetArtifact(config.ScanType, config.BuildDescriptorFile, opts, cmd)
		if err != nil {
			return err
		}
		gav, err := artifact.GetCoordinates()
		if err != nil {
			return err
		}

		nameTmpl := `{{list .GroupID .ArtifactID | join "-" | trimAll "-"}}`
		pName, pVer := versioning.DetermineProjectCoordinates(nameTmpl, config.DefaultVersioningModel, gav)
		if config.ProjectName == "" {
			log.Entry().Infof("Resolved project name '%s' from descriptor file", pName)
			config.ProjectName = pName
		}
		if config.ProductVersion == "" {
			log.Entry().Infof("Resolved project version '%s' from descriptor file", pVer)
			config.ProductVersion = pVer
		}
	}

	// Get product token if user did not specify one at runtime
	if config.ProductToken == "" {
		log.Entry().Infof("Attempting to resolve product token for product '%s'..", config.ProductName)
		product, err := sys.GetProductByName(config.ProductName)
		if err != nil {
			return err
		}
		if product != nil {
			log.Entry().Infof("Resolved product token: '%s'..", product.Token)
			config.ProductToken = product.Token
		}
	}

	// Get project token  if user did not specify one at runtime
	if config.ProjectToken == "" {
		log.Entry().Infof("Attempting to resolve project token for project '%s'..", config.ProjectName)
		fullProjName := fmt.Sprintf("%s - %s", config.ProjectName, config.ProductVersion)
		projectToken, err := sys.GetProjectToken(config.ProductToken, fullProjName)
		if err != nil {
			return err
		}
		if projectToken != "" {
			log.Entry().Infof("Resolved project token: '%s'..", projectToken)
			config.ProjectToken = projectToken
		}
	}
	return nil
}

func triggerWhitesourceScan(cmd *command.Command, config *ScanOptions) error {
	switch config.ScanType {
	case "npm":
		// Execute whitesource scan with
		if err := executeNpmScan(config, cmd); err != nil {
			return err
		}
	default:
		// Download the unified agent jar file if one does not exist
		if err := downloadAgent(config, cmd); err != nil {
			return err
		}

		// Auto generate a config file based on the working directory's contents.
		// TODO/NOTE: Currently this scans the UA jar file as a dependency since it is downloaded beforehand
		if err := autoGenerateWhitesourceConfig(config, cmd); err != nil {
			return err
		}

		// Execute whitesource scan with unified agent jar file
		if err := executeUAScan(config, cmd); err != nil {
			return err
		}
	}
	return nil
}

// executeUAScan
// Executes a scan with the Whitesource Unified Agent
// returns stdout buffer of the unified agent for token extraction in case of multi-module gradle project
func executeUAScan(config *ScanOptions, cmd *command.Command) error {
	return cmd.RunExecutable("java", "-jar", config.AgentFileName, "-d", ".", "-c", config.ConfigFilePath,
		"-apiKey", config.OrgToken, "-userKey", config.UserToken, "-project", config.ProjectName,
		"-product", config.ProductName, "-productVersion", config.ProductVersion)
}

// executeNpmScan
// generates a configuration file whitesource.config.json with appropriate values from config,
// installs whitesource yarn plugin and executes the scan
func executeNpmScan(config *ScanOptions, cmd *command.Command) error {
	npmConfig := []byte(fmt.Sprintf(`{
		"apiKey": "%s",
		"userKey": "%s",
		"checkPolicies": true,
		"productName": "%s",
		"projectName": "%s",
		"productVer": "%s",
		"devDep": true
	}`, config.OrgToken, config.UserToken, config.ProductName, config.ProjectName, config.ProductVersion))
	if err := ioutil.WriteFile("whitesource.config.json", npmConfig, 0644); err != nil {
		return err
	}
	if err := cmd.RunExecutable("yarn", "global", "add", "whitesource"); err != nil {
		return err
	}
	if err := cmd.RunExecutable("yarn", "install"); err != nil {
		return err
	}
	if err := cmd.RunExecutable("whitesource", "yarn"); err != nil {
		return err
	}
	return nil
}

// checkSecurityViolations: checks security violations and fails build is severity limit is crossed
func checkSecurityViolations(config *ScanOptions, sys *System) error {
	severeVulnerabilities := 0

	// convert config.CvssSeverityLimit to float64
	cvssSeverityLimit, err := strconv.ParseFloat(config.CvssSeverityLimit, 64)
	if err != nil {
		return err
	}

	// get project alerts (vulnerabilities)
	alerts, err := sys.GetProjectAlerts(config.ProjectToken)
	if err != nil {
		return err
	}

	// https://github.com/SAP/jenkins-library/blob/master/vars/whitesourceExecuteScan.groovy#L537
	for _, alert := range alerts {
		vuln := alert.Vulnerability
		if (vuln.Score >= cvssSeverityLimit || vuln.CVSS3Score >= cvssSeverityLimit) && cvssSeverityLimit >= 0 {
			severeVulnerabilities++
		}
	}

	//https://github.com/SAP/jenkins-library/blob/master/vars/whitesourceExecuteScan.groovy#L547
	nonSevereVulnerabilities := len(alerts) - severeVulnerabilities
	if nonSevereVulnerabilities > 0 {
		log.Entry().Warnf("WARNING: %v Open Source Software Security vulnerabilities with "+
			"CVSS score below threshold %s detected in project %s.", nonSevereVulnerabilities,
			config.CvssSeverityLimit, config.ProjectName)
	} else if len(alerts) == 0 {
		log.Entry().Infof("No Open Source Software Security vulnerabilities detected in project %s",
			config.ProjectName)
	}

	// https://github.com/SAP/jenkins-library/blob/master/vars/whitesourceExecuteScan.groovy#L558
	if severeVulnerabilities > 0 {
		return fmt.Errorf("%v Open Source Software Security vulnerabilities with CVSS score greater "+
			"or equal to %s detected in project %s",
			severeVulnerabilities, config.CvssSeverityLimit, config.ProjectName)
	}
	return nil
}

// pollProjectStatus polls project LastUpdateTime until it reflects the most recent scan
func pollProjectStatus(config *ScanOptions, sys *System) error {
	currentTime := time.Now()
	for {
		project, err := sys.GetProjectVitals(config.ProjectToken)
		if err != nil {
			return err
		}

		// Make sure the project was updated in whitesource backend before downloading any reports
		lastUpdatedTime, err := time.Parse("2006-01-02 15:04:05 +0000", project.LastUpdateDate)
		if currentTime.Sub(lastUpdatedTime) < 10*time.Second {
			//done polling
			break
		}
		log.Entry().Info("time since project was last updated > 10 seconds, polling status...")
		time.Sleep(5 * time.Second)
	}
	return nil
}

// downloadReports downloads a project's risk and vulnerability reports
func downloadReports(config *ScanOptions, sys *System) ([]piperutils.Path, error) {
	utils := piperutils.Files{}

	// Project was scanned, now we need to wait for Whitesource backend to propagate the changes
	if err := pollProjectStatus(config, sys); err != nil {
		return nil, err
	}

	if err := utils.MkdirAll(config.ReportDirectoryName, 0777); err != nil {
		return nil, err
	}
	vulnPath, err := downloadVulnerabilityReport(config, sys)
	if err != nil {
		return nil, err
	}
	riskPath, err := downloadRiskReport(config, sys)
	if err != nil {
		return nil, err
	}
	return []piperutils.Path{*vulnPath, *riskPath}, nil
}

func downloadVulnerabilityReport(config *ScanOptions, sys *System) (*piperutils.Path, error) {
	utils := piperutils.Files{}
	if err := utils.MkdirAll(config.ReportDirectoryName, 0777); err != nil {
		return nil, err
	}

	reportBytes, err := sys.GetProjectVulnerabilityReport(config.ProjectToken, config.VulnerabilityReportFormat)
	if err != nil {
		return nil, err
	}

	// Write report to file
	rptFileName := fmt.Sprintf("%s-vulnerability-report.%s", config.ProjectName, config.VulnerabilityReportFormat)
	rptFileName = filepath.Join(config.ReportDirectoryName, rptFileName)
	if err := ioutil.WriteFile(rptFileName, reportBytes, 0644); err != nil {
		return nil, err
	}

	log.Entry().Infof("Successfully downloaded vulnerability report to %s", rptFileName)
	pathName := fmt.Sprintf("%s Vulnerability Report", config.ProjectName)
	return &piperutils.Path{Name: pathName, Target: rptFileName}, nil
}

func downloadRiskReport(config *ScanOptions, sys *System) (*piperutils.Path, error) {
	reportBytes, err := sys.GetProjectRiskReport(config.ProjectToken)
	if err != nil {
		return nil, err
	}

	rptFileName := fmt.Sprintf("%s-risk-report.pdf", config.ProjectName)
	rptFileName = filepath.Join(config.ReportDirectoryName, rptFileName)
	if err := ioutil.WriteFile(rptFileName, reportBytes, 0644); err != nil {
		return nil, err
	}

	log.Entry().Infof("Successfully downloaded risk report to %s", rptFileName)
	pathName := fmt.Sprintf("%s PDF Risk Report", config.ProjectName)
	return &piperutils.Path{Name: pathName, Target: rptFileName}, nil
}

// downloadAgent: Downloads the unified agent jar file if one does not exist
func downloadAgent(config *ScanOptions, cmd *command.Command) error {
	agentFile := config.AgentFileName
	if !fileExists(agentFile) {
		if err := cmd.RunExecutable("curl", "-L", config.AgentDownloadURL, "-o", agentFile); err != nil {
			return err
		}
	}
	return nil
}

// autoGenerateWhitesourceConfig
// Auto generate a config file based on the current directory structure, renames it to user specified configFilePath
// Generated file name will be 'wss-generated-file.config'
func autoGenerateWhitesourceConfig(config *ScanOptions, cmd *command.Command) error {
	// TODO: Should we rely on -detect, or set the parameters manually?
	if err := cmd.RunExecutable("java", "-jar", config.AgentFileName, "-d", ".", "-detect"); err != nil {
		return err
	}

	// Rename generated config file to config.ConfigFilePath parameter
	if err := os.Rename("wss-generated-file.config", config.ConfigFilePath); err != nil {
		return err
	}

	// Append aggregateModules=true parameter to config file (consolidates multi-module projects into one)
	f, err := os.OpenFile(config.ConfigFilePath, os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}
	defer f.Close()

	// Append additional config parameters to prevent multiple projects being generated
	cfg := fmt.Sprintf("gradle.aggregateModules=true\nmaven.aggregateModules=true\ngradle.localRepositoryPath=.gradle\nmaven.m2RepositoryPath=.m2\nexcludes=%s", config.Excludes)
	if _, err = f.WriteString(cfg); err != nil {
		return err
	}

	// archiveExtractionDepth=0
	if err := cmd.RunExecutable("sed", "-ir", `s/^[#]*\s*archiveExtractionDepth=.*/archiveExtractionDepth=0/`,
		config.ConfigFilePath); err != nil {
		return err
	}

	// config.Includes defaults to "**/*.java **/*.jar **/*.py **/*.go **/*.js **/*.ts"
	regex := fmt.Sprintf(`s/^[#]*\s*includes=.*/includes="%s"/`, config.Includes)
	if err := cmd.RunExecutable("sed", "-ir", regex, config.ConfigFilePath); err != nil {
		return err
	}

	return nil
}

func aggregateVersionWideLibraries(sys *System, config *ScanOptions) error {
	log.Entry().Infof("Aggregating list of libraries used for all projects with version: %s", config.ProductVersion)

	projects, err := sys.GetProjectsMetaInfo(config.ProductToken)
	if err != nil {
		return err
	}

	versionWideLibraries := map[string][]whitesource.Library{} // maps project name to slice of libraries
	for _, project := range projects {
		projectVersion := strings.Split(project.Name, " - ")[1]
		projectName := strings.Split(project.Name, " - ")[0]
		if projectVersion == config.ProductVersion {
			libs, err := sys.GetProjectLibraryLocations(project.Token)
			if err != nil {
				return err
			}
			log.Entry().Infof("Found project: %s with %v libraries.", project.Name, len(libs))
			versionWideLibraries[projectName] = libs
		}
	}
	if err := newLibraryCSVReport(versionWideLibraries, config); err != nil {
		return err
	}
	return nil
}

func aggregateVersionWideVulnerabilities(sys *System, config *ScanOptions) error {
	log.Entry().Infof("Aggregating list of vulnerabilities for all projects with version: %s", config.ProductVersion)

	projects, err := sys.GetProjectsMetaInfo(config.ProductToken)
	if err != nil {
		return err
	}

	var versionWideAlerts []whitesource.Alert // all alerts for a given project version
	projectNames := ``                        // holds all project tokens considered a part of the report for debugging
	for _, project := range projects {
		projectVersion := strings.Split(project.Name, " - ")[1]
		if projectVersion == config.ProductVersion {
			projectNames += project.Name + "\n"
			alerts, err := sys.GetProjectAlerts(project.Token)
			if err != nil {
				return err
			}
			log.Entry().Infof("Found project: %s with %v vulnerabilities.", project.Name, len(alerts))
			versionWideAlerts = append(versionWideAlerts, alerts...)
		}
	}

	if err := ioutil.WriteFile("whitesource-reports/project-names-aggregated.txt", []byte(projectNames), 0777); err != nil {
		return err
	}
	if err := newVulnerabilityExcelReport(versionWideAlerts, config); err != nil {
		return err
	}
	return nil
}

// outputs an slice of alerts to an excel file
func newVulnerabilityExcelReport(alerts []whitesource.Alert, config *ScanOptions) error {
	file := excelize.NewFile()
	streamWriter, err := file.NewStreamWriter("Sheet1")
	if err != nil {
		return err
	}
	styleID, err := file.NewStyle(`{"font":{"color":"#777777"}}`)
	if err != nil {
		return err
	}
	if err := streamWriter.SetRow("A1", []interface{}{excelize.Cell{StyleID: styleID, Value: "Severity"}}); err != nil {
		return err
	}
	if err := streamWriter.SetRow("B1", []interface{}{excelize.Cell{StyleID: styleID, Value: "Library"}}); err != nil {
		return err
	}
	if err := streamWriter.SetRow("C1", []interface{}{excelize.Cell{StyleID: styleID, Value: "Vulnerability ID"}}); err != nil {
		return err
	}
	if err := streamWriter.SetRow("D1", []interface{}{excelize.Cell{StyleID: styleID, Value: "Project"}}); err != nil {
		return err
	}
	if err := streamWriter.SetRow("E1", []interface{}{excelize.Cell{StyleID: styleID, Value: "Resolution"}}); err != nil {
		return err
	}

	for i, alert := range alerts {
		row := make([]interface{}, 5)
		vuln := alert.Vulnerability
		row[0] = vuln.Severity
		row[1] = alert.Library.Filename
		row[2] = vuln.Level
		row[3] = alert.Project
		row[4] = vuln.FixResolutionText
		cell, _ := excelize.CoordinatesToCellName(1, i+2)
		if err := streamWriter.SetRow(cell, row); err != nil {
			fmt.Println(err)
		}
	}
	if err := streamWriter.Flush(); err != nil {
		return err
	}

	utils := piperutils.Files{}
	if err := utils.MkdirAll(config.ReportDirectoryName, 0777); err != nil {
		return err
	}

	fileName := fmt.Sprintf("%s/vulnerabilities-%s.xlsx", config.ReportDirectoryName, time.Now().Format("2006-01-01 15:00:00"))
	if err := file.SaveAs(fileName); err != nil {
		return err
	}
	return nil
}

// outputs an slice of libraries to an excel file based on projects with version == config.ProductVersion
func newLibraryCSVReport(libraries map[string][]whitesource.Library, config *ScanOptions) error {
	output := "Library Name, Project Name\n"
	for projectName, libraries := range libraries {
		log.Entry().Infof("Writing %v libraries for project %s to excel report..", len(libraries), projectName)
		for _, library := range libraries {
			output += library.Name + ", " + projectName + "\n"
		}
	}

	// Ensure reporting directory exists
	utils := piperutils.Files{}
	if err := utils.MkdirAll(config.ReportDirectoryName, 0777); err != nil {
		return err
	}

	// Write result to file
	fileName := fmt.Sprintf("%s/libraries-%s.csv", config.ReportDirectoryName, time.Now().Format("2006-01-01 15:00:00"))
	if err := ioutil.WriteFile(fileName, []byte(output), 0777); err != nil {
		return err
	}
	return nil
}