jenkins/sap-jenkins-library
1
0
Fork 0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-01-28 05:47:08 +02:00
Code Issues Releases Activity

Refactor whitesourceExecuteScan, fix polling, error handling (#2036)

Browse Source
This commit is contained in:
Stephan Aßmus 2020-09-18 11:54:45 +02:00 committed by GitHub
parent 13d1b562bf
commit 33e6e13787
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 1077 additions and 504 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

374
cmd/whitesourceExecuteScan.go
View File

@ -1,8 +1,11 @@
package cmd
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strconv"
@ -11,51 +14,112 @@ import (
"github.com/360EntSecGroup-Skylar/excelize/v2"
"github.com/SAP/jenkins-library/pkg/command"
piperhttp "github.com/SAP/jenkins-library/pkg/http"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/SAP/jenkins-library/pkg/telemetry"
"github.com/SAP/jenkins-library/pkg/versioning"
"github.com/SAP/jenkins-library/pkg/whitesource"
ws "github.com/SAP/jenkins-library/pkg/whitesource"
)
// just to make the lines less long
type ScanOptions = whitesourceExecuteScanOptions
type System = whitesource.System
func whitesourceExecuteScan(config ScanOptions, telemetry *telemetry.CustomData) {
// reroute cmd output to logging framework
c := command.Command{}
c.Stdout(log.Writer())
c.Stderr(log.Writer())
// whitesource defines the functions that are expected by the step implementation to
// be available from the whitesource system.
type whitesource interface {
GetProductByName(productName string) (ws.Product, error)
GetProjectsMetaInfo(productToken string) ([]ws.Project, error)
GetProjectToken(productToken, projectName string) (string, error)
GetProjectByToken(projectToken string) (ws.Project, error)
GetProjectRiskReport(projectToken string) ([]byte, error)
GetProjectVulnerabilityReport(projectToken string, format string) ([]byte, error)
GetProjectAlerts(projectToken string) ([]ws.Alert, error)
GetProjectLibraryLocations(projectToken string) ([]ws.Library, error)
}
sys := whitesource.NewSystem(config.ServiceURL, config.OrgToken, config.UserToken)
if err := resolveProjectIdentifiers(&c, sys, &config); err != nil {
type whitesourceUtils interface {
Stdout(out io.Writer)
Stderr(err io.Writer)
RunExecutable(executable string, params ...string) error
DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error
Chdir(path string) error
Getwd() (string, error)
MkdirAll(path string, perm os.FileMode) error
FileExists(path string) (bool, error)
FileRead(path string) ([]byte, error)
FileWrite(path string, content []byte, perm os.FileMode) error
FileRemove(path string) error
FileRename(oldPath, newPath string) error
RemoveAll(path string) error
FileOpen(name string, flag int, perm os.FileMode) (*os.File, error)
GetArtifactCoordinates(config *ScanOptions) (versioning.Coordinates, error)
}
type whitesourceUtilsBundle struct {
*piperhttp.Client
*command.Command
*piperutils.Files
}
func (w *whitesourceUtilsBundle) GetArtifactCoordinates(config *ScanOptions) (versioning.Coordinates, error) {
opts := &versioning.Options{
ProjectSettingsFile: config.ProjectSettingsFile,
GlobalSettingsFile: config.GlobalSettingsFile,
M2Path: config.M2Path,
}
artifact, err := versioning.GetArtifact(config.BuildTool, config.BuildDescriptorFile, opts, w)
if err != nil {
return nil, err
}
return artifact.GetCoordinates()
}
func newWhitesourceUtils() *whitesourceUtilsBundle {
utils := whitesourceUtilsBundle{
Client: &piperhttp.Client{},
Command: &command.Command{},
Files: &piperutils.Files{},
}
// Reroute cmd output to logging framework
utils.Stdout(log.Writer())
utils.Stderr(log.Writer())
return &utils
}
func whitesourceExecuteScan(config ScanOptions, _ *telemetry.CustomData) {
utils := newWhitesourceUtils()
sys := ws.NewSystem(config.ServiceURL, config.OrgToken, config.UserToken)
if err := resolveProjectIdentifiers(&config, utils, sys); err != nil {
log.Entry().WithError(err).Fatal("step execution failed on resolving project identifiers")
}
// Generate a vulnerability report for all projects with version = config.ProjectVersion
if config.AggregateVersionWideReport {
if err := aggregateVersionWideLibraries(sys, &config); err != nil {
if err := aggregateVersionWideLibraries(&config, utils, sys); err != nil {
log.Entry().WithError(err).Fatal("step execution failed on aggregating version wide libraries")
}
if err := aggregateVersionWideVulnerabilities(sys, &config); err != nil {
if err := aggregateVersionWideVulnerabilities(&config, utils, sys); err != nil {
log.Entry().WithError(err).Fatal("step execution failed on aggregating version wide vulnerabilities")
}
} else {
if err := runWhitesourceScan(&config, sys, telemetry, &c); err != nil {
if err := runWhitesourceScan(&config, utils, sys); err != nil {
log.Entry().WithError(err).Fatal("step execution failed on executing whitesource scan")
}
}
}
func runWhitesourceScan(config *ScanOptions, sys *System, _ *telemetry.CustomData, cmd *command.Command) error {
func runWhitesourceScan(config *ScanOptions, utils whitesourceUtils, sys whitesource) error {
// Start the scan
if err := triggerWhitesourceScan(cmd, config); err != nil {
if err := executeScan(config, utils); err != nil {
return err
}
// Scan finished: we need to resolve project token again if the project was just created.
if err := resolveProjectIdentifiers(cmd, sys, config); err != nil {
if err := resolveProjectIdentifiers(config, utils, sys); err != nil {
return err
}
@ -65,16 +129,24 @@ func runWhitesourceScan(config *ScanOptions, sys *System, _ *telemetry.CustomDat
log.Entry().Infof("Project Token: %s", config.ProjectToken)
log.Entry().Info("-----------------------------------------------------")
if config.Reporting || config.SecurityVulnerabilities {
// Project was scanned. We need to wait for WhiteSource backend to propagate the changes
// before downloading any reports or check security vulnerabilities.
if err := pollProjectStatus(config, sys); err != nil {
return err
}
}
if config.Reporting {
paths, err := downloadReports(config, sys)
paths, err := downloadReports(config, utils, sys)
if err != nil {
return err
}
piperutils.PersistReportsAndLinks("whitesourceExecuteScan", "", nil, paths)
}
// Check for security vulnerabilities and fail the build if cvssSeverityLimit threshold is crossed
if config.SecurityVulnerabilities {
// Check for security vulnerabilities and fail the build if cvssSeverityLimit threshold is crossed
if err := checkSecurityViolations(config, sys); err != nil {
return err
}
@ -82,27 +154,23 @@ func runWhitesourceScan(config *ScanOptions, sys *System, _ *telemetry.CustomDat
return nil
}
func resolveProjectIdentifiers(cmd *command.Command, sys *System, config *ScanOptions) error {
func resolveProjectIdentifiers(config *ScanOptions, utils whitesourceUtils, sys whitesource) error {
if config.ProjectName == "" || config.ProductVersion == "" {
opts := &versioning.Options{}
artifact, err := versioning.GetArtifact(config.ScanType, config.BuildDescriptorFile, opts, cmd)
coordinates, err := utils.GetArtifactCoordinates(config)
if err != nil {
return err
}
gav, err := artifact.GetCoordinates()
if err != nil {
return err
return fmt.Errorf("failed to get build artifact description: %w", err)
}
nameTmpl := `{{list .GroupID .ArtifactID | join "-" | trimAll "-"}}`
pName, pVer := versioning.DetermineProjectCoordinates(nameTmpl, config.DefaultVersioningModel, gav)
name, version := versioning.DetermineProjectCoordinates(nameTmpl, config.VersioningModel, coordinates)
if config.ProjectName == "" {
log.Entry().Infof("Resolved project name '%s' from descriptor file", pName)
config.ProjectName = pName
log.Entry().Infof("Resolved project name '%s' from descriptor file", name)
config.ProjectName = name
}
if config.ProductVersion == "" {
log.Entry().Infof("Resolved project version '%s' from descriptor file", pVer)
config.ProductVersion = pVer
log.Entry().Infof("Resolved product version '%s' from descriptor file with versioning '%s'",
version, config.VersioningModel)
config.ProductVersion = version
}
}
@ -113,10 +181,8 @@ func resolveProjectIdentifiers(cmd *command.Command, sys *System, config *ScanOp
if err != nil {
return err
}
if product != nil {
log.Entry().Infof("Resolved product token: '%s'..", product.Token)
config.ProductToken = product.Token
}
log.Entry().Infof("Resolved product token: '%s'..", product.Token)
config.ProductToken = product.Token
}
// Get project token if user did not specify one at runtime
@ -127,99 +193,150 @@ func resolveProjectIdentifiers(cmd *command.Command, sys *System, config *ScanOp
if err != nil {
return err
}
if projectToken != "" {
log.Entry().Infof("Resolved project token: '%s'..", projectToken)
config.ProjectToken = projectToken
if projectToken == "" {
return fmt.Errorf("failed to resolve project token for '%s' and product token %s",
config.ProjectName, config.ProductToken)
}
log.Entry().Infof("Resolved project token: '%s'..", projectToken)
config.ProjectToken = projectToken
}
return nil
}
func triggerWhitesourceScan(cmd *command.Command, config *ScanOptions) error {
// executeScan executes different types of scans depending on the scanType parameter.
// The default is to download the Unified Agent and use it to perform the scan.
func executeScan(config *ScanOptions, utils whitesourceUtils) error {
if config.ScanType == "" {
config.ScanType = config.BuildTool
}
switch config.ScanType {
case "npm":
// Execute whitesource scan with
if err := executeNpmScan(config, cmd); err != nil {
// Execute scan with whitesource yarn plugin
if err := executeYarnScan(config, utils); err != nil {
return err
}
default:
// Download the unified agent jar file if one does not exist
if err := downloadAgent(config, cmd); err != nil {
if err := downloadAgent(config, utils); err != nil {
return err
}
// Auto generate a config file based on the working directory's contents.
// TODO/NOTE: Currently this scans the UA jar file as a dependency since it is downloaded beforehand
if err := autoGenerateWhitesourceConfig(config, cmd); err != nil {
if err := autoGenerateWhitesourceConfig(config, utils); err != nil {
return err
}
// Execute whitesource scan with unified agent jar file
if err := executeUAScan(config, cmd); err != nil {
if err := executeUAScan(config, utils); err != nil {
return err
}
}
return nil
}
// executeUAScan
// Executes a scan with the Whitesource Unified Agent
// returns stdout buffer of the unified agent for token extraction in case of multi-module gradle project
func executeUAScan(config *ScanOptions, cmd *command.Command) error {
return cmd.RunExecutable("java", "-jar", config.AgentFileName, "-d", ".", "-c", config.ConfigFilePath,
// executeUAScan executes a scan with the Whitesource Unified Agent.
func executeUAScan(config *ScanOptions, utils whitesourceUtils) error {
return utils.RunExecutable("java", "-jar", config.AgentFileName, "-d", ".", "-c", config.ConfigFilePath,
"-apiKey", config.OrgToken, "-userKey", config.UserToken, "-project", config.ProjectName,
"-product", config.ProductName, "-productVersion", config.ProductVersion)
}
// executeNpmScan
// generates a configuration file whitesource.config.json with appropriate values from config,
// installs whitesource yarn plugin and executes the scan
func executeNpmScan(config *ScanOptions, cmd *command.Command) error {
npmConfig := []byte(fmt.Sprintf(`{
"apiKey": "%s",
"userKey": "%s",
"checkPolicies": true,
"productName": "%s",
"projectName": "%s",
"productVer": "%s",
"devDep": true
}`, config.OrgToken, config.UserToken, config.ProductName, config.ProjectName, config.ProductVersion))
if err := ioutil.WriteFile("whitesource.config.json", npmConfig, 0644); err != nil {
const whiteSourceConfig = "whitesource.config.json"
func setValueAndLogChange(config map[string]interface{}, key string, value interface{}) {
oldValue, exists := config[key]
if exists && oldValue != value {
log.Entry().Infof("overwriting '%s' in %s: %v -> %v", key, whiteSourceConfig, oldValue, value)
}
config[key] = value
}
func writeWhitesourceConfigJSON(config *ScanOptions, utils whitesourceUtils, devDep, ignoreLsErrors bool) error {
var npmConfig = make(map[string]interface{})
exists, _ := utils.FileExists(whiteSourceConfig)
if exists {
fileContents, err := utils.FileRead(whiteSourceConfig)
if err != nil {
return fmt.Errorf("file '%s' already exists, but could not be read: %w", whiteSourceConfig, err)
}
err = json.Unmarshal(fileContents, &npmConfig)
if err != nil {
return fmt.Errorf("file '%s' already exists, but could not be parsed: %w", whiteSourceConfig, err)
}
log.Entry().Infof("The file '%s' already exists in the project. Changed config details will be logged.",
whiteSourceConfig)
}
npmConfig["apiKey"] = config.OrgToken
npmConfig["userKey"] = config.UserToken
setValueAndLogChange(npmConfig, "checkPolicies", true)
setValueAndLogChange(npmConfig, "productName", config.ProductName)
setValueAndLogChange(npmConfig, "projectName", config.ProjectName)
setValueAndLogChange(npmConfig, "productVer", config.ProductVersion)
setValueAndLogChange(npmConfig, "devDep", devDep)
setValueAndLogChange(npmConfig, "ignoreNpmLsErrors", ignoreLsErrors)
jsonBuffer, err := json.Marshal(npmConfig)
if err != nil {
return fmt.Errorf("failed to generate '%s': %w", whiteSourceConfig, err)
}
err = utils.FileWrite(whiteSourceConfig, jsonBuffer, 0644)
if err != nil {
return fmt.Errorf("failed to write '%s': %w", whiteSourceConfig, err)
}
return nil
}
// executeYarnScan generates a configuration file whitesource.config.json with appropriate values from config,
// installs whitesource yarn plugin and executes the scan.
func executeYarnScan(config *ScanOptions, utils whitesourceUtils) error {
if err := writeWhitesourceConfigJSON(config, utils, true, false); err != nil {
return err
}
if err := cmd.RunExecutable("yarn", "global", "add", "whitesource"); err != nil {
defer func() { _ = utils.FileRemove(whiteSourceConfig) }()
if err := utils.RunExecutable("yarn", "global", "add", "whitesource"); err != nil {
return err
}
if err := cmd.RunExecutable("yarn", "install"); err != nil {
if err := utils.RunExecutable("yarn", "install"); err != nil {
return err
}
if err := cmd.RunExecutable("whitesource", "yarn"); err != nil {
if err := utils.RunExecutable("whitesource", "yarn"); err != nil {
return err
}
return nil
}
// checkSecurityViolations: checks security violations and fails build is severity limit is crossed
func checkSecurityViolations(config *ScanOptions, sys *System) error {
severeVulnerabilities := 0
// checkSecurityViolations checks security violations and returns an error if the configured severity limit is crossed.
func checkSecurityViolations(config *ScanOptions, sys whitesource) error {
// convert config.CvssSeverityLimit to float64
cvssSeverityLimit, err := strconv.ParseFloat(config.CvssSeverityLimit, 64)
if err != nil {
return err
log.SetErrorCategory(log.ErrorConfiguration)
return fmt.Errorf("failed to parse parameter cvssSeverityLimit (%s) "+
"as floating point number: %w", config.CvssSeverityLimit, err)
}
// get project alerts (vulnerabilities)
alerts, err := sys.GetProjectAlerts(config.ProjectToken)
if err != nil {
return err
return fmt.Errorf("failed to retrieve project alerts from Whitesource: %w", err)
}
severeVulnerabilities := 0
// https://github.com/SAP/jenkins-library/blob/master/vars/whitesourceExecuteScan.groovy#L537
for _, alert := range alerts {
vuln := alert.Vulnerability
if (vuln.Score >= cvssSeverityLimit || vuln.CVSS3Score >= cvssSeverityLimit) && cvssSeverityLimit >= 0 {
log.Entry().Infof("Vulnerability with Score %v / CVSS3Score %v treated as severe",
vuln.Score, vuln.CVSS3Score)
severeVulnerabilities++
} else {
log.Entry().Infof("Ignoring vulnerability with Score %v / CVSS3Score %v",
vuln.Score, vuln.CVSS3Score)
}
}
@ -243,56 +360,65 @@ func checkSecurityViolations(config *ScanOptions, sys *System) error {
return nil
}
// pollProjectStatus polls project LastUpdateTime until it reflects the most recent scan
func pollProjectStatus(config *ScanOptions, sys *System) error {
currentTime := time.Now()
// pollProjectStatus polls project LastUpdateDate until it reflects the most recent scan
func pollProjectStatus(config *ScanOptions, sys whitesource) error {
return blockUntilProjectIsUpdated(config, sys, time.Now(), 20*time.Second, 20*time.Second, 15*time.Minute)
}
const whitesourceDateTimeLayout = "2006-01-02 15:04:05 -0700"
// blockUntilProjectIsUpdated polls the project LastUpdateDate until it is newer than the given time stamp
// or no older than maxAge relative to the given time stamp.
func blockUntilProjectIsUpdated(config *ScanOptions, sys whitesource, currentTime time.Time, maxAge, timeBetweenPolls, maxWaitTime time.Duration) error {
startTime := time.Now()
for {
project, err := sys.GetProjectVitals(config.ProjectToken)
project, err := sys.GetProjectByToken(config.ProjectToken)
if err != nil {
return err
}
// Make sure the project was updated in whitesource backend before downloading any reports
lastUpdatedTime, err := time.Parse("2006-01-02 15:04:05 +0000", project.LastUpdateDate)
if currentTime.Sub(lastUpdatedTime) < 10*time.Second {
//done polling
break
if project.LastUpdateDate == "" {
log.Entry().Infof("last updated time missing from project metadata, retrying")
} else {
lastUpdatedTime, err := time.Parse(whitesourceDateTimeLayout, project.LastUpdateDate)
if err != nil {
return fmt.Errorf("failed to parse last updated time (%s) of Whitesource project: %w",
project.LastUpdateDate, err)
}
age := currentTime.Sub(lastUpdatedTime)
if age < maxAge {
//done polling
break
}
log.Entry().Infof("time since project was last updated %v > %v, polling status...", age, maxAge)
}
log.Entry().Info("time since project was last updated > 10 seconds, polling status...")
time.Sleep(5 * time.Second)
if time.Now().Sub(startTime) > maxWaitTime {
return fmt.Errorf("timeout while waiting for Whitesource scan results to be reflected in service")
}
time.Sleep(timeBetweenPolls)
}
return nil
}
// downloadReports downloads a project's risk and vulnerability reports
func downloadReports(config *ScanOptions, sys *System) ([]piperutils.Path, error) {
utils := piperutils.Files{}
// Project was scanned, now we need to wait for Whitesource backend to propagate the changes
if err := pollProjectStatus(config, sys); err != nil {
func downloadReports(config *ScanOptions, utils whitesourceUtils, sys whitesource) ([]piperutils.Path, error) {
if err := utils.MkdirAll(config.ReportDirectoryName, os.ModePerm); err != nil {
return nil, err
}
if err := utils.MkdirAll(config.ReportDirectoryName, 0777); err != nil {
return nil, err
}
vulnPath, err := downloadVulnerabilityReport(config, sys)
vulnPath, err := downloadVulnerabilityReport(config, utils, sys)
if err != nil {
return nil, err
}
riskPath, err := downloadRiskReport(config, sys)
riskPath, err := downloadRiskReport(config, utils, sys)
if err != nil {
return nil, err
}
return []piperutils.Path{*vulnPath, *riskPath}, nil
}
func downloadVulnerabilityReport(config *ScanOptions, sys *System) (*piperutils.Path, error) {
utils := piperutils.Files{}
if err := utils.MkdirAll(config.ReportDirectoryName, 0777); err != nil {
return nil, err
}
func downloadVulnerabilityReport(config *ScanOptions, utils whitesourceUtils, sys whitesource) (*piperutils.Path, error) {
reportBytes, err := sys.GetProjectVulnerabilityReport(config.ProjectToken, config.VulnerabilityReportFormat)
if err != nil {
return nil, err
@ -301,7 +427,7 @@ func downloadVulnerabilityReport(config *ScanOptions, sys *System) (*piperutils.
// Write report to file
rptFileName := fmt.Sprintf("%s-vulnerability-report.%s", config.ProjectName, config.VulnerabilityReportFormat)
rptFileName = filepath.Join(config.ReportDirectoryName, rptFileName)
if err := ioutil.WriteFile(rptFileName, reportBytes, 0644); err != nil {
if err := utils.FileWrite(rptFileName, reportBytes, 0644); err != nil {
return nil, err
}
@ -310,7 +436,7 @@ func downloadVulnerabilityReport(config *ScanOptions, sys *System) (*piperutils.
return &piperutils.Path{Name: pathName, Target: rptFileName}, nil
}
func downloadRiskReport(config *ScanOptions, sys *System) (*piperutils.Path, error) {
func downloadRiskReport(config *ScanOptions, utils whitesourceUtils, sys whitesource) (*piperutils.Path, error) {
reportBytes, err := sys.GetProjectRiskReport(config.ProjectToken)
if err != nil {
return nil, err
@ -318,7 +444,7 @@ func downloadRiskReport(config *ScanOptions, sys *System) (*piperutils.Path, err
rptFileName := fmt.Sprintf("%s-risk-report.pdf", config.ProjectName)
rptFileName = filepath.Join(config.ReportDirectoryName, rptFileName)
if err := ioutil.WriteFile(rptFileName, reportBytes, 0644); err != nil {
if err := utils.FileWrite(rptFileName, reportBytes, 0644); err != nil {
return nil, err
}
@ -327,12 +453,14 @@ func downloadRiskReport(config *ScanOptions, sys *System) (*piperutils.Path, err
return &piperutils.Path{Name: pathName, Target: rptFileName}, nil
}
// downloadAgent: Downloads the unified agent jar file if one does not exist
func downloadAgent(config *ScanOptions, cmd *command.Command) error {
// downloadAgent downloads the unified agent jar file if one does not exist
func downloadAgent(config *ScanOptions, utils whitesourceUtils) error {
agentFile := config.AgentFileName
if !fileExists(agentFile) {
if err := cmd.RunExecutable("curl", "-L", config.AgentDownloadURL, "-o", agentFile); err != nil {
return err
err := utils.DownloadFile(config.AgentDownloadURL, agentFile, nil, nil)
if err != nil {
return fmt.Errorf("failed to download unified agent from URL '%s' to file '%s': %w",
config.AgentDownloadURL, agentFile, err)
}
}
return nil
@ -341,23 +469,23 @@ func downloadAgent(config *ScanOptions, cmd *command.Command) error {
// autoGenerateWhitesourceConfig
// Auto generate a config file based on the current directory structure, renames it to user specified configFilePath
// Generated file name will be 'wss-generated-file.config'
func autoGenerateWhitesourceConfig(config *ScanOptions, cmd *command.Command) error {
func autoGenerateWhitesourceConfig(config *ScanOptions, utils whitesourceUtils) error {
// TODO: Should we rely on -detect, or set the parameters manually?
if err := cmd.RunExecutable("java", "-jar", config.AgentFileName, "-d", ".", "-detect"); err != nil {
if err := utils.RunExecutable("java", "-jar", config.AgentFileName, "-d", ".", "-detect"); err != nil {
return err
}
// Rename generated config file to config.ConfigFilePath parameter
if err := os.Rename("wss-generated-file.config", config.ConfigFilePath); err != nil {
if err := utils.FileRename("wss-generated-file.config", config.ConfigFilePath); err != nil {
return err
}
// Append aggregateModules=true parameter to config file (consolidates multi-module projects into one)
f, err := os.OpenFile(config.ConfigFilePath, os.O_APPEND|os.O_WRONLY, 0600)
f, err := utils.FileOpen(config.ConfigFilePath, os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
return err
}
defer f.Close()
defer func() { _ = f.Close() }()
// Append additional config parameters to prevent multiple projects being generated
cfg := fmt.Sprintf("gradle.aggregateModules=true\nmaven.aggregateModules=true\ngradle.localRepositoryPath=.gradle\nmaven.m2RepositoryPath=.m2\nexcludes=%s", config.Excludes)
@ -366,21 +494,21 @@ func autoGenerateWhitesourceConfig(config *ScanOptions, cmd *command.Command) er
}
// archiveExtractionDepth=0
if err := cmd.RunExecutable("sed", "-ir", `s/^[#]*\s*archiveExtractionDepth=.*/archiveExtractionDepth=0/`,
if err := utils.RunExecutable("sed", "-ir", `s/^[#]*\s*archiveExtractionDepth=.*/archiveExtractionDepth=0/`,
config.ConfigFilePath); err != nil {
return err
}
// config.Includes defaults to "**/*.java **/*.jar **/*.py **/*.go **/*.js **/*.ts"
regex := fmt.Sprintf(`s/^[#]*\s*includes=.*/includes="%s"/`, config.Includes)
if err := cmd.RunExecutable("sed", "-ir", regex, config.ConfigFilePath); err != nil {
if err := utils.RunExecutable("sed", "-ir", regex, config.ConfigFilePath); err != nil {
return err
}
return nil
}
func aggregateVersionWideLibraries(sys *System, config *ScanOptions) error {
func aggregateVersionWideLibraries(config *ScanOptions, utils whitesourceUtils, sys whitesource) error {
log.Entry().Infof("Aggregating list of libraries used for all projects with version: %s", config.ProductVersion)
projects, err := sys.GetProjectsMetaInfo(config.ProductToken)
@ -388,7 +516,7 @@ func aggregateVersionWideLibraries(sys *System, config *ScanOptions) error {
return err
}
versionWideLibraries := map[string][]whitesource.Library{} // maps project name to slice of libraries
versionWideLibraries := map[string][]ws.Library{} // maps project name to slice of libraries
for _, project := range projects {
projectVersion := strings.Split(project.Name, " - ")[1]
projectName := strings.Split(project.Name, " - ")[0]
@ -401,13 +529,13 @@ func aggregateVersionWideLibraries(sys *System, config *ScanOptions) error {
versionWideLibraries[projectName] = libs
}
}
if err := newLibraryCSVReport(versionWideLibraries, config); err != nil {
if err := newLibraryCSVReport(versionWideLibraries, config, utils); err != nil {
return err
}
return nil
}
func aggregateVersionWideVulnerabilities(sys *System, config *ScanOptions) error {
func aggregateVersionWideVulnerabilities(config *ScanOptions, utils whitesourceUtils, sys whitesource) error {
log.Entry().Infof("Aggregating list of vulnerabilities for all projects with version: %s", config.ProductVersion)
projects, err := sys.GetProjectsMetaInfo(config.ProductToken)
@ -415,8 +543,8 @@ func aggregateVersionWideVulnerabilities(sys *System, config *ScanOptions) error
return err
}
var versionWideAlerts []whitesource.Alert // all alerts for a given project version
projectNames := `` // holds all project tokens considered a part of the report for debugging
var versionWideAlerts []ws.Alert // all alerts for a given project version
projectNames := `` // holds all project tokens considered a part of the report for debugging
for _, project := range projects {
projectVersion := strings.Split(project.Name, " - ")[1]
if projectVersion == config.ProductVersion {
@ -433,14 +561,14 @@ func aggregateVersionWideVulnerabilities(sys *System, config *ScanOptions) error
if err := ioutil.WriteFile("whitesource-reports/project-names-aggregated.txt", []byte(projectNames), 0777); err != nil {
return err
}
if err := newVulnerabilityExcelReport(versionWideAlerts, config); err != nil {
if err := newVulnerabilityExcelReport(versionWideAlerts, config, utils); err != nil {
return err
}
return nil
}
// outputs an slice of alerts to an excel file
func newVulnerabilityExcelReport(alerts []whitesource.Alert, config *ScanOptions) error {
func newVulnerabilityExcelReport(alerts []ws.Alert, config *ScanOptions, utils whitesourceUtils) error {
file := excelize.NewFile()
streamWriter, err := file.NewStreamWriter("Sheet1")
if err != nil {
@ -483,7 +611,6 @@ func newVulnerabilityExcelReport(alerts []whitesource.Alert, config *ScanOptions
return err
}
utils := piperutils.Files{}
if err := utils.MkdirAll(config.ReportDirectoryName, 0777); err != nil {
return err
}
@ -496,7 +623,7 @@ func newVulnerabilityExcelReport(alerts []whitesource.Alert, config *ScanOptions
}
// outputs an slice of libraries to an excel file based on projects with version == config.ProductVersion
func newLibraryCSVReport(libraries map[string][]whitesource.Library, config *ScanOptions) error {
func newLibraryCSVReport(libraries map[string][]ws.Library, config *ScanOptions, utils whitesourceUtils) error {
output := "Library Name, Project Name\n"
for projectName, libraries := range libraries {
log.Entry().Infof("Writing %v libraries for project %s to excel report..", len(libraries), projectName)
@ -506,7 +633,6 @@ func newLibraryCSVReport(libraries map[string][]whitesource.Library, config *Sca
}
// Ensure reporting directory exists
utils := piperutils.Files{}
if err := utils.MkdirAll(config.ReportDirectoryName, 0777); err != nil {
return err
}

145
cmd/whitesourceExecuteScan_generated.go
View File

@ -14,8 +14,9 @@ import (
)
type whitesourceExecuteScanOptions struct {
BuildTool string `json:"buildTool,omitempty"`
BuildDescriptorFile string `json:"buildDescriptorFile,omitempty"`
DefaultVersioningModel string `json:"defaultVersioningModel,omitempty"`
VersioningModel string `json:"versioningModel,omitempty"`
CreateProductFromPipeline bool `json:"createProductFromPipeline,omitempty"`
SecurityVulnerabilities bool `json:"securityVulnerabilities,omitempty"`
Timeout string `json:"timeout,omitempty"`
@ -32,7 +33,7 @@ type whitesourceExecuteScanOptions struct {
UserToken string `json:"userToken,omitempty"`
LicensingVulnerabilities bool `json:"licensingVulnerabilities,omitempty"`
AgentFileName string `json:"agentFileName,omitempty"`
EmailAddressesOfInitialProductAdmins string `json:"emailAddressesOfInitialProductAdmins,omitempty"`
EmailAddressesOfInitialProductAdmins []string `json:"emailAddressesOfInitialProductAdmins,omitempty"`
ProductVersion string `json:"productVersion,omitempty"`
JreDownloadURL string `json:"jreDownloadUrl,omitempty"`
ProductName string `json:"productName,omitempty"`
@ -46,6 +47,10 @@ type whitesourceExecuteScanOptions struct {
Excludes string `json:"excludes,omitempty"`
ProductToken string `json:"productToken,omitempty"`
AgentParameters string `json:"agentParameters,omitempty"`
ProjectSettingsFile string `json:"projectSettingsFile,omitempty"`
GlobalSettingsFile string `json:"globalSettingsFile,omitempty"`
M2Path string `json:"m2Path,omitempty"`
DefaultNpmRegistry string `json:"defaultNpmRegistry,omitempty"`
}
// WhitesourceExecuteScanCommand BETA
@ -119,8 +124,9 @@ check and additional Free and Open Source Software Publicly Known Vulnerabilitie
}
func addWhitesourceExecuteScanFlags(cmd *cobra.Command, stepConfig *whitesourceExecuteScanOptions) {
cmd.Flags().StringVar(&stepConfig.BuildTool, "buildTool", os.Getenv("PIPER_buildTool"), "Defines the tool which is used for building the artifact.")
cmd.Flags().StringVar(&stepConfig.BuildDescriptorFile, "buildDescriptorFile", os.Getenv("PIPER_buildDescriptorFile"), "Explicit path to the build descriptor file.")
cmd.Flags().StringVar(&stepConfig.DefaultVersioningModel, "defaultVersioningModel", `major`, "The default project versioning model used in case `projectVersion` parameter is empty for creating the version based on the build descriptor version to report results in Whitesource, can be one of `'major'`, `'major-minor'`, `'semantic'`, `'full'`")
cmd.Flags().StringVar(&stepConfig.VersioningModel, "versioningModel", `major`, "The default project versioning model used in case `projectVersion` parameter is empty for creating the version based on the build descriptor version to report results in Whitesource, can be one of `'major'`, `'major-minor'`, `'semantic'`, `'full'`")
cmd.Flags().BoolVar(&stepConfig.CreateProductFromPipeline, "createProductFromPipeline", true, "Whether to create the related WhiteSource product on the fly based on the supplied pipeline configuration.")
cmd.Flags().BoolVar(&stepConfig.SecurityVulnerabilities, "securityVulnerabilities", true, "Whether security compliance is considered and reported as part of the assessment.")
cmd.Flags().StringVar(&stepConfig.Timeout, "timeout", `0`, "Timeout in seconds until a HTTP call is forcefully terminated.")
@ -130,28 +136,33 @@ func addWhitesourceExecuteScanFlags(cmd *cobra.Command, stepConfig *whitesourceE
cmd.Flags().BoolVar(&stepConfig.AggregateVersionWideReport, "aggregateVersionWideReport", false, "This does not run a scan, instead just generated a report for all projects with projectVersion = config.ProductVersion")
cmd.Flags().StringVar(&stepConfig.VulnerabilityReportFormat, "vulnerabilityReportFormat", `xlsx`, "Format of the file the vulnerability report is written to.")
cmd.Flags().StringVar(&stepConfig.ParallelLimit, "parallelLimit", `15`, "Limit of parallel jobs being run at once in case of `scanType: 'mta'` based scenarios, defaults to `15`.")
cmd.Flags().BoolVar(&stepConfig.Reporting, "reporting", true, "Whether assessment is being done at all, defaults to `true`.")
cmd.Flags().BoolVar(&stepConfig.Reporting, "reporting", true, "Whether assessment is being done at all, defaults to `true`")
cmd.Flags().StringVar(&stepConfig.ServiceURL, "serviceUrl", `https://saas.whitesourcesoftware.com/api`, "URL to the WhiteSource server API used for communication.")
cmd.Flags().StringSliceVar(&stepConfig.BuildDescriptorExcludeList, "buildDescriptorExcludeList", []string{``}, "List of build descriptors and therefore modules to exclude from the scan and assessment activities.")
cmd.Flags().StringSliceVar(&stepConfig.BuildDescriptorExcludeList, "buildDescriptorExcludeList", []string{}, "List of build descriptors and therefore modules to exclude from the scan and assessment activities.")
cmd.Flags().StringVar(&stepConfig.OrgToken, "orgToken", os.Getenv("PIPER_orgToken"), "WhiteSource token identifying your organization.")
cmd.Flags().StringVar(&stepConfig.UserToken, "userToken", os.Getenv("PIPER_userToken"), "WhiteSource token identifying the user executing the scan")
cmd.Flags().BoolVar(&stepConfig.LicensingVulnerabilities, "licensingVulnerabilities", true, "Whether license compliance is considered and reported as part of the assessment.")
cmd.Flags().StringVar(&stepConfig.AgentFileName, "agentFileName", `wss-unified-agent.jar`, "Locally used name for the Unified Agent jar file after download.")
cmd.Flags().StringVar(&stepConfig.EmailAddressesOfInitialProductAdmins, "emailAddressesOfInitialProductAdmins", `[]`, "The list of email addresses to assign as product admins for newly created WhiteSource products.")
cmd.Flags().StringSliceVar(&stepConfig.EmailAddressesOfInitialProductAdmins, "emailAddressesOfInitialProductAdmins", []string{}, "The list of email addresses to assign as product admins for newly created WhiteSource products.")
cmd.Flags().StringVar(&stepConfig.ProductVersion, "productVersion", os.Getenv("PIPER_productVersion"), "Version of the WhiteSource product to be created and used for results aggregation, usually determined automatically.")
cmd.Flags().StringVar(&stepConfig.JreDownloadURL, "jreDownloadUrl", os.Getenv("PIPER_jreDownloadUrl"), "URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent.")
cmd.Flags().StringVar(&stepConfig.ProductName, "productName", os.Getenv("PIPER_productName"), "Name of the WhiteSource product to be created and used for results aggregation.")
cmd.Flags().StringVar(&stepConfig.ProjectName, "projectName", `{{list .GroupID .ArtifactID | join "-" | trimAll "-"}}`, "The project used for reporting results in Whitesource")
cmd.Flags().StringVar(&stepConfig.ProjectName, "projectName", os.Getenv("PIPER_projectName"), "The project used for reporting results in Whitesource")
cmd.Flags().StringVar(&stepConfig.ProjectToken, "projectToken", os.Getenv("PIPER_projectToken"), "Project token to execute scan on")
cmd.Flags().StringVar(&stepConfig.VulnerabilityReportTitle, "vulnerabilityReportTitle", `WhiteSource Security Vulnerability Report`, "Title of vulnerability report written during the assessment phase.")
cmd.Flags().StringVar(&stepConfig.InstallCommand, "installCommand", os.Getenv("PIPER_installCommand"), "Install command that can be used to populate the default docker image for some scenarios.")
cmd.Flags().StringVar(&stepConfig.ScanType, "scanType", os.Getenv("PIPER_scanType"), "Type of development stack used to implement the solution.")
cmd.Flags().StringVar(&stepConfig.CvssSeverityLimit, "cvssSeverityLimit", `-1`, "Limit of tollerable CVSS v3 score upon assessment and in consequence fails the build, defaults to `-1`.")
cmd.Flags().StringVar(&stepConfig.Includes, "includes", `**\/src\/main\/**\/*.java **\/*.py **\/*.go **\/*.js **\/*.ts`, "Space separated list of file path patterns to include in the scan, slashes must be escaped for sed")
cmd.Flags().StringVar(&stepConfig.CvssSeverityLimit, "cvssSeverityLimit", `-1`, "Limit of tolerable CVSS v3 score upon assessment and in consequence fails the build, defaults to `-1`.")
cmd.Flags().StringVar(&stepConfig.Includes, "includes", `**\/src\/main\/**\/*.java **\/*.py **\/*.go **\/*.js **\/*.ts`, "Space separated list of file path patterns to include in the scan, slashes must be escaped for sed.")
cmd.Flags().StringVar(&stepConfig.Excludes, "excludes", `tests/**/*.py **/src/test/**/*.java`, "Space separated list of file path patterns to exclude in the scan")
cmd.Flags().StringVar(&stepConfig.ProductToken, "productToken", os.Getenv("PIPER_productToken"), "Token of the WhiteSource product to be created and used for results aggregation, usually determined automatically.")
cmd.Flags().StringVar(&stepConfig.AgentParameters, "agentParameters", ``, "Additional parameters passed to the Unified Agent command line.")
cmd.Flags().StringVar(&stepConfig.AgentParameters, "agentParameters", os.Getenv("PIPER_agentParameters"), "Additional parameters passed to the Unified Agent command line.")
cmd.Flags().StringVar(&stepConfig.ProjectSettingsFile, "projectSettingsFile", os.Getenv("PIPER_projectSettingsFile"), "Path to the mvn settings file that should be used as project settings file.")
cmd.Flags().StringVar(&stepConfig.GlobalSettingsFile, "globalSettingsFile", os.Getenv("PIPER_globalSettingsFile"), "Path to the mvn settings file that should be used as global settings file.")
cmd.Flags().StringVar(&stepConfig.M2Path, "m2Path", os.Getenv("PIPER_m2Path"), "Path to the location of the local repository that should be used.")
cmd.Flags().StringVar(&stepConfig.DefaultNpmRegistry, "defaultNpmRegistry", os.Getenv("PIPER_defaultNpmRegistry"), "URL of the npm registry to use. Defaults to https://registry.npmjs.org/")
cmd.MarkFlagRequired("buildTool")
cmd.MarkFlagRequired("orgToken")
cmd.MarkFlagRequired("userToken")
cmd.MarkFlagRequired("productName")
@ -168,12 +179,17 @@ func whitesourceExecuteScanMetadata() config.StepData {
Inputs: config.StepInputs{
Parameters: []config.StepParameters{
{
Name: "buildDescriptorFile",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Name: "buildTool",
ResourceRef: []config.ResourceReference{
{
Name: "commonPipelineEnvironment",
Param: "buildTool",
},
},
Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: true,
Aliases: []config.Alias{},
},
{
Name: "buildDescriptorFile",
@ -184,52 +200,12 @@ func whitesourceExecuteScanMetadata() config.StepData {
Aliases: []config.Alias{},
},
{
Name: "buildDescriptorFile",
Name: "versioningModel",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Scope: []string{"PARAMETERS", "STAGES", "STEPS", "GENERAL"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "buildDescriptorFile",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "buildDescriptorFile",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "buildDescriptorFile",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "buildDescriptorFile",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "defaultVersioningModel",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Aliases: []config.Alias{{Name: "defaultVersioningModel"}},
},
{
Name: "createProductFromPipeline",
@ -373,17 +349,22 @@ func whitesourceExecuteScanMetadata() config.StepData {
Name: "emailAddressesOfInitialProductAdmins",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Type: "[]string",
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "productVersion",
ResourceRef: []config.ResourceReference{},
Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Name: "productVersion",
ResourceRef: []config.ResourceReference{
{
Name: "commonPipelineEnvironment",
Param: "artifactVersion",
},
},
Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "jreDownloadUrl",
@ -481,6 +462,38 @@ func whitesourceExecuteScanMetadata() config.StepData {
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "projectSettingsFile",
ResourceRef: []config.ResourceReference{},
Scope: []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{{Name: "maven/projectSettingsFile"}},
},
{
Name: "globalSettingsFile",
ResourceRef: []config.ResourceReference{},
Scope: []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{{Name: "maven/globalSettingsFile"}},
},
{
Name: "m2Path",
ResourceRef: []config.ResourceReference{},
Scope: []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{{Name: "maven/m2Path"}},
},
{
Name: "defaultNpmRegistry",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "GENERAL", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{{Name: "npm/defaultNpmRegistry"}},
},
},
},
},

373
cmd/whitesourceExecuteScan_test.go Normal file
View File

@ -0,0 +1,373 @@
package cmd
import (
"encoding/json"
"fmt"
"github.com/SAP/jenkins-library/pkg/mock"
"github.com/SAP/jenkins-library/pkg/versioning"
ws "github.com/SAP/jenkins-library/pkg/whitesource"
"github.com/stretchr/testify/assert"
"net/http"
"os"
"path/filepath"
"testing"
"time"
)
type whitesourceSystemMock struct {
productName string
products []ws.Product
projects []ws.Project
alerts []ws.Alert
libraries []ws.Library
riskReport []byte
vulnerabilityReport []byte
}
func (m *whitesourceSystemMock) GetProductByName(productName string) (ws.Product, error) {
for _, product := range m.products {
if product.Name == productName {
return product, nil
}
}
return ws.Product{}, fmt.Errorf("no product with name '%s' found in Whitesource", productName)
}
func (m *whitesourceSystemMock) GetProjectsMetaInfo(productToken string) ([]ws.Project, error) {
return m.projects, nil
}
func (m *whitesourceSystemMock) GetProjectToken(productToken, projectName string) (string, error) {
return "mock-project-token", nil
}
func (m *whitesourceSystemMock) GetProjectByToken(projectToken string) (ws.Project, error) {
for _, project := range m.projects {
if project.Token == projectToken {
return project, nil
}
}
return ws.Project{}, fmt.Errorf("no project with token '%s' found in Whitesource", projectToken)
}
func (m *whitesourceSystemMock) GetProjectRiskReport(projectToken string) ([]byte, error) {
return m.riskReport, nil
}
func (m *whitesourceSystemMock) GetProjectVulnerabilityReport(projectToken string, format string) ([]byte, error) {
_, err := m.GetProjectByToken(projectToken)
if err != nil {
return nil, err
}
if m.vulnerabilityReport == nil {
return nil, fmt.Errorf("no report available")
}
return m.vulnerabilityReport, nil
}
func (m *whitesourceSystemMock) GetProjectAlerts(projectToken string) ([]ws.Alert, error) {
return m.alerts, nil
}
func (m *whitesourceSystemMock) GetProjectLibraryLocations(projectToken string) ([]ws.Library, error) {
return m.libraries, nil
}
var mockLibrary = ws.Library{
Name: "mock-library",
Filename: "mock-library-file",
Version: "mock-library-version",
Project: "mock-project",
}
func newWhitesourceSystemMock(lastUpdateDate string) *whitesourceSystemMock {
return &whitesourceSystemMock{
productName: "mock-product",
products: []ws.Product{
{
Name: "mock-product",
Token: "mock-product-token",
CreationDate: "last-thursday",
LastUpdateDate: lastUpdateDate,
},
},
projects: []ws.Project{
{
ID: 42,
Name: "mock-project",
PluginName: "mock-plugin-name",
Token: "mock-project-token",
UploadedBy: "MrBean",
CreationDate: "last-thursday",
LastUpdateDate: lastUpdateDate,
},
},
alerts: []ws.Alert{
{
Vulnerability: ws.Vulnerability{},
Library: mockLibrary,
Project: "mock-project",
CreationDate: "last-thursday",
},
},
libraries: []ws.Library{mockLibrary},
riskReport: []byte("mock-risk-report"),
vulnerabilityReport: []byte("mock-vulnerability-report"),
}
}
type whitesourceCoordinatesMock struct {
GroupID string
ArtifactID string
Version string
}
type downloadedFile struct {
sourceURL string
filePath string
}
type whitesourceUtilsMock struct {
*mock.FilesMock
*mock.ExecMockRunner
coordinates whitesourceCoordinatesMock
downloadedFiles []downloadedFile
}
func (w *whitesourceUtilsMock) DownloadFile(url, filename string, _ http.Header, _ []*http.Cookie) error {
w.downloadedFiles = append(w.downloadedFiles, downloadedFile{sourceURL: url, filePath: filename})
return nil
}
func (w *whitesourceUtilsMock) FileOpen(name string, flag int, perm os.FileMode) (*os.File, error) {
return nil, fmt.Errorf("FileOpen() is unsupported by the mock implementation")
}
func (w *whitesourceUtilsMock) RemoveAll(path string) error {
// TODO: Implement in FS Mock
return nil
}
func (w *whitesourceUtilsMock) GetArtifactCoordinates(_ *ScanOptions) (versioning.Coordinates, error) {
return w.coordinates, nil
}
func (w *whitesourceUtilsMock) FindPackageJSONFiles(_ *ScanOptions) ([]string, error) {
matches, _ := w.Glob("**/package.json")
return matches, nil
}
func (w *whitesourceUtilsMock) InstallAllNPMDependencies(_ *ScanOptions, _ []string) error {
return nil
}
func newWhitesourceUtilsMock() *whitesourceUtilsMock {
return &whitesourceUtilsMock{
FilesMock: &mock.FilesMock{},
ExecMockRunner: &mock.ExecMockRunner{},
coordinates: whitesourceCoordinatesMock{
GroupID: "mock-group-id",
ArtifactID: "mock-artifact-id",
Version: "1.0.42",
},
}
}
func TestResolveProjectIdentifiers(t *testing.T) {
t.Run("happy path", func(t *testing.T) {
// init
config := ScanOptions{
ScanType: "mta",
VersioningModel: "major",
ProductName: "mock-product",
}
utilsMock := newWhitesourceUtilsMock()
systemMock := newWhitesourceSystemMock("ignored")
// test
err := resolveProjectIdentifiers(&config, utilsMock, systemMock)
// assert
if assert.NoError(t, err) {
assert.Equal(t, "mock-group-id-mock-artifact-id", config.ProjectName)
assert.Equal(t, "1", config.ProductVersion)
assert.Equal(t, "mock-project-token", config.ProjectToken)
assert.Equal(t, "mock-product-token", config.ProductToken)
}
})
t.Run("product not found", func(t *testing.T) {
// init
config := ScanOptions{
ScanType: "mta",
VersioningModel: "major",
ProductName: "does-not-exist",
}
utilsMock := newWhitesourceUtilsMock()
systemMock := newWhitesourceSystemMock("ignored")
// test
err := resolveProjectIdentifiers(&config, utilsMock, systemMock)
// assert
assert.EqualError(t, err, "no product with name 'does-not-exist' found in Whitesource")
})
}
func TestBlockUntilProjectIsUpdated(t *testing.T) {
t.Parallel()
t.Run("already new enough", func(t *testing.T) {
// init
nowString := "2010-05-30 00:15:00 +0100"
now, err := time.Parse(whitesourceDateTimeLayout, nowString)
if err != nil {
t.Fatalf(err.Error())
}
lastUpdatedDate := "2010-05-30 00:15:01 +0100"
systemMock := newWhitesourceSystemMock(lastUpdatedDate)
config := &ScanOptions{
ProjectToken: systemMock.projects[0].Token,
}
// test
err = blockUntilProjectIsUpdated(config, systemMock, now, 2*time.Second, 1*time.Second, 2*time.Second)
// assert
assert.NoError(t, err)
})
t.Run("timeout while polling", func(t *testing.T) {
// init
nowString := "2010-05-30 00:15:00 +0100"
now, err := time.Parse(whitesourceDateTimeLayout, nowString)
if err != nil {
t.Fatalf(err.Error())
}
lastUpdatedDate := "2010-05-30 00:07:00 +0100"
systemMock := newWhitesourceSystemMock(lastUpdatedDate)
config := &ScanOptions{
ProjectToken: systemMock.projects[0].Token,
}
// test
err = blockUntilProjectIsUpdated(config, systemMock, now, 2*time.Second, 1*time.Second, 1*time.Second)
// assert
if assert.Error(t, err) {
assert.Contains(t, err.Error(), "timeout while waiting")
}
})
t.Run("timeout while polling, no update time", func(t *testing.T) {
// init
nowString := "2010-05-30 00:15:00 +0100"
now, err := time.Parse(whitesourceDateTimeLayout, nowString)
if err != nil {
t.Fatalf(err.Error())
}
systemMock := newWhitesourceSystemMock("")
config := &ScanOptions{
ProjectToken: systemMock.projects[0].Token,
}
// test
err = blockUntilProjectIsUpdated(config, systemMock, now, 2*time.Second, 1*time.Second, 1*time.Second)
// assert
if assert.Error(t, err) {
assert.Contains(t, err.Error(), "timeout while waiting")
}
})
}
func TestDownloadReports(t *testing.T) {
t.Parallel()
t.Run("happy path", func(t *testing.T) {
// init
config := &ScanOptions{
ProjectToken: "mock-project-token",
ProjectName: "mock-project",
ReportDirectoryName: "report-dir",
VulnerabilityReportFormat: "txt",
}
utils := newWhitesourceUtilsMock()
system := newWhitesourceSystemMock("2010-05-30 00:15:00 +0100")
// test
paths, err := downloadReports(config, utils, system)
// assert
if assert.NoError(t, err) && assert.Len(t, paths, 2) {
vPath := filepath.Join("report-dir", "mock-project-vulnerability-report.txt")
assert.True(t, utils.HasWrittenFile(vPath))
vContent, _ := utils.FileRead(vPath)
assert.Equal(t, []byte("mock-vulnerability-report"), vContent)
rPath := filepath.Join("report-dir", "mock-project-risk-report.pdf")
assert.True(t, utils.HasWrittenFile(rPath))
rContent, _ := utils.FileRead(rPath)
assert.Equal(t, []byte("mock-risk-report"), rContent)
}
})
t.Run("invalid project token", func(t *testing.T) {
// init
config := &ScanOptions{
ProjectToken: "<invalid>",
}
utils := newWhitesourceUtilsMock()
system := newWhitesourceSystemMock("2010-05-30 00:15:00 +0100")
// test
path, err := downloadReports(config, utils, system)
// assert
assert.EqualError(t, err, "no project with token '<invalid>' found in Whitesource")
assert.Nil(t, path)
})
}
func TestWriteWhitesourceConfigJSON(t *testing.T) {
config := &ScanOptions{
OrgToken: "org-token",
UserToken: "user-token",
ProductName: "mock-product",
ProjectName: "mock-project",
ProductVersion: "42",
}
expected := make(map[string]interface{})
expected["apiKey"] = "org-token"
expected["userKey"] = "user-token"
expected["checkPolicies"] = true
expected["productName"] = "mock-product"
expected["projectName"] = "mock-project"
expected["productVer"] = "42"
expected["devDep"] = true
expected["ignoreNpmLsErrors"] = true
t.Parallel()
t.Run("write config from scratch", func(t *testing.T) {
// init
utils := newWhitesourceUtilsMock()
// test
err := writeWhitesourceConfigJSON(config, utils, true, true)
// assert
if assert.NoError(t, err) && assert.True(t, utils.HasWrittenFile(whiteSourceConfig)) {
contents, _ := utils.FileRead(whiteSourceConfig)
actual := make(map[string]interface{})
_ = json.Unmarshal(contents, &actual)
assert.Equal(t, expected, actual)
}
})
t.Run("extend and merge config", func(t *testing.T) {
// init
initial := make(map[string]interface{})
initial["checkPolicies"] = false
initial["productName"] = "mock-product"
initial["productVer"] = "41"
initial["unknown"] = "preserved"
encoded, _ := json.Marshal(initial)
utils := newWhitesourceUtilsMock()
utils.AddFile(whiteSourceConfig, encoded)
// test
err := writeWhitesourceConfigJSON(config, utils, true, true)
// assert
if assert.NoError(t, err) && assert.True(t, utils.HasWrittenFile(whiteSourceConfig)) {
contents, _ := utils.FileRead(whiteSourceConfig)
actual := make(map[string]interface{})
_ = json.Unmarshal(contents, &actual)
mergedExpected := expected
mergedExpected["unknown"] = "preserved"
assert.Equal(t, mergedExpected, actual)
}
})
}

36
pkg/mock/fileUtils.go
View File

@ -256,11 +256,43 @@ func (f *FilesMock) FileRemove(path string) error {
return nil
}
// FileRename changes the path under which content is associated in the virtual file system.
// Only leaf-entries are supported as of yet.
func (f *FilesMock) FileRename(oldPath, newPath string) error {
if f.files == nil {
return fmt.Errorf("the file '%s' does not exist: %w", oldPath, os.ErrNotExist)
}
oldAbsPath := f.toAbsPath(oldPath)
props, exists := f.files[oldAbsPath]
// If there is no leaf-entry in the map, path may be a directory.
// We only support renaming leaf-entries for now.
if !exists {
return fmt.Errorf("renaming file '%s' is not supported, since it does not exist, "+
"or is not a leaf-entry", oldPath)
}
if oldPath == newPath {
return nil
}
newAbsPath := f.toAbsPath(newPath)
_, exists = f.files[newAbsPath]
// Fail if the target path already exists
if exists {
return fmt.Errorf("cannot rename '%s', target path '%s' already exists", oldPath, newPath)
}
delete(f.files, oldAbsPath)
f.files[newAbsPath] = props
return nil
}
// MkdirAll creates a directory in the in-memory file system, so that this path is established to exist.
func (f *FilesMock) MkdirAll(path string, mode os.FileMode) error {
// NOTE: FilesMock could be extended to have a set of paths for which MkdirAll should fail.
// This is why AddDir() exists separately, to differentiate the notion of setting up the mocking
// versus implementing the methods from Files.
// This is why AddDirWithMode() exists separately, to differentiate the notion of setting up
// the mocking versus implementing the methods from Files.
f.AddDirWithMode(path, mode)
return nil
}

69
pkg/mock/fileUtils_test.go
View File

@ -230,6 +230,75 @@ func TestFilesMockFileRemove(t *testing.T) {
})
}
func TestFilesMockFileRename(t *testing.T) {
t.Parallel()
t.Run("fail to rename non-existing file (no init)", func(t *testing.T) {
files := FilesMock{}
oldPath := filepath.Join("foo", "bar")
newPath := filepath.Join("foo", "baz")
err := files.FileRename(oldPath, newPath)
assert.EqualError(t, err, "the file '"+oldPath+"' does not exist: file does not exist")
})
t.Run("fail to rename non-existing file", func(t *testing.T) {
files := FilesMock{}
files.AddDir("triggers/initialization")
oldPath := filepath.Join("foo", "bar")
newPath := filepath.Join("foo", "baz")
err := files.FileRename(oldPath, newPath)
assert.EqualError(t, err,
"renaming file '"+oldPath+"' is not supported, since it does not exist, or is not a leaf-entry")
})
t.Run("fail to rename non-existing file, even no-op", func(t *testing.T) {
files := FilesMock{}
files.AddDir("triggers/initialization")
oldPath := filepath.Join("foo", "bar")
newPath := oldPath
err := files.FileRename(oldPath, newPath)
assert.EqualError(t, err,
"renaming file '"+oldPath+"' is not supported, since it does not exist, or is not a leaf-entry")
})
t.Run("success to rename dir (no-op)", func(t *testing.T) {
files := FilesMock{}
oldPath := filepath.Join("foo", "bar")
files.AddDir(oldPath)
newPath := oldPath
err := files.FileRename(oldPath, newPath)
assert.NoError(t, err)
assert.True(t, files.HasFile(newPath))
})
t.Run("success to rename dir", func(t *testing.T) {
files := FilesMock{}
oldPath := filepath.Join("foo", "bar")
files.AddDir(oldPath)
newPath := filepath.Join("foo", "baz")
err := files.FileRename(oldPath, newPath)
assert.NoError(t, err)
assert.True(t, files.HasFile(newPath))
assert.False(t, files.HasFile(oldPath))
})
t.Run("success to rename file", func(t *testing.T) {
files := FilesMock{}
oldPath := filepath.Join("foo", "bar")
files.AddFile(oldPath, []byte("dummy contents"))
newPath := filepath.Join("foo", "baz")
err := files.FileRename(oldPath, newPath)
assert.NoError(t, err)
assert.True(t, files.HasFile(newPath))
assert.False(t, files.HasFile(oldPath))
})
t.Run("fail to rename file, already exists", func(t *testing.T) {
files := FilesMock{}
oldPath := filepath.Join("foo", "bar")
newPath := filepath.Join("foo", "baz")
files.AddFile(oldPath, []byte("dummy contents"))
files.AddFile(newPath, []byte("dummy contents"))
err := files.FileRename(oldPath, newPath)
assert.EqualError(t, err, "cannot rename '"+oldPath+"', target path '"+newPath+"' already exists")
assert.True(t, files.HasFile(newPath))
assert.True(t, files.HasFile(oldPath))
})
}
func TestFilesMockGetwd(t *testing.T) {
t.Parallel()
t.Run("test root", func(t *testing.T) {

34
pkg/piperutils/FileUtils.go
View File

@ -78,18 +78,18 @@ func (f Files) Copy(src, dst string) (int64, error) {
if err != nil {
return 0, err
}
defer source.Close()
defer func() { _ = source.Close() }()
destination, err := os.Create(dst)
if err != nil {
return 0, err
}
defer destination.Close()
defer func() { _ = destination.Close() }()
nBytes, err := io.Copy(destination, source)
return nBytes, err
}
//Chmod ...
//Chmod is a wrapper for os.Chmod().
func (f Files) Chmod(path string, mode os.FileMode) error {
return os.Chmod(path, mode)
}
@ -126,7 +126,7 @@ func Unzip(src, dest string) ([]string, error) {
if err != nil {
return filenames, err
}
defer r.Close()
defer func() { _ = r.Close() }()
for _, f := range r.File {
@ -142,7 +142,10 @@ func Unzip(src, dest string) ([]string, error) {
if f.FileInfo().IsDir() {
// Make Folder
os.MkdirAll(fpath, os.ModePerm)
err := os.MkdirAll(fpath, os.ModePerm)
if err != nil {
return filenames, fmt.Errorf("failed to create directory: %w", err)
}
continue
}
@ -164,8 +167,8 @@ func Unzip(src, dest string) ([]string, error) {
_, err = io.Copy(outFile, rc)
// Close the file without defer to close before next iteration of loop
outFile.Close()
rc.Close()
_ = outFile.Close()
_ = rc.Close()
if err != nil {
return filenames, err
@ -189,16 +192,31 @@ func (f Files) FileWrite(path string, content []byte, perm os.FileMode) error {
return ioutil.WriteFile(path, content, perm)
}
// FileRemove is a wrapper for os.FileRemove().
// FileRemove is a wrapper for os.Remove().
func (f Files) FileRemove(path string) error {
return os.Remove(path)
}
// FileRename is a wrapper for os.Rename().
func (f Files) FileRename(oldPath, newPath string) error {
return os.Rename(oldPath, newPath)
}
// FileOpen is a wrapper for os.OpenFile().
func (f *Files) FileOpen(name string, flag int, perm os.FileMode) (*os.File, error) {
return os.OpenFile(name, flag, perm)
}
// MkdirAll is a wrapper for os.MkdirAll().
func (f Files) MkdirAll(path string, perm os.FileMode) error {
return os.MkdirAll(path, perm)
}
// RemoveAll is a wrapper for os.RemoveAll().
func (f Files) RemoveAll(path string) error {
return os.RemoveAll(path)
}
// Glob is a wrapper for doublestar.Glob().
func (f Files) Glob(pattern string) (matches []string, err error) {
return doublestar.Glob(pattern)

194
pkg/whitesource/whitesource.go
View File

@ -72,21 +72,21 @@ type Request struct {
Format string `json:"format,omitempty"`
}
// System defines a WhiteSource system including respective tokens (e.g. org token, user token)
// System defines a WhiteSource System including respective tokens (e.g. org token, user token)
type System struct {
HTTPClient piperhttp.Sender
OrgToken string
ServerURL string
UserToken string
httpClient piperhttp.Sender
orgToken string
serverURL string
userToken string
}
// NewSystem constructs a new system instance
// NewSystem constructs a new System instance
func NewSystem(serverURL, orgToken, userToken string) *System {
return &System{
ServerURL: serverURL,
OrgToken: orgToken,
UserToken: userToken,
HTTPClient: &piperhttp.Client{},
serverURL: serverURL,
orgToken: orgToken,
userToken: userToken,
httpClient: &piperhttp.Client{},
}
}
@ -102,21 +102,16 @@ func (s *System) GetProductsMetaInfo() ([]Product, error) {
RequestType: "getOrganizationProductVitals",
}
respBody, err := s.sendRequest(req)
err := s.sendRequestAndDecodeJSON(req, &wsResponse)
if err != nil {
return wsResponse.ProductVitals, errors.Wrap(err, "WhiteSource request failed")
}
err = json.Unmarshal(respBody, &wsResponse)
if err != nil {
return wsResponse.ProductVitals, errors.Wrap(err, "failed to parse WhiteSource response")
}
return wsResponse.ProductVitals, nil
}
// GetMetaInfoForProduct retrieves meta information for a specific WhiteSource product
func (s *System) GetMetaInfoForProduct(productName string) (Product, error) {
// GetProductByName retrieves meta information for a specific WhiteSource product
func (s *System) GetProductByName(productName string) (Product, error) {
products, err := s.GetProductsMetaInfo()
if err != nil {
return Product{}, errors.Wrap(err, "failed to retrieve WhiteSource products")
@ -131,7 +126,7 @@ func (s *System) GetMetaInfoForProduct(productName string) (Product, error) {
return Product{}, fmt.Errorf("product '%v' not found in WhiteSource", productName)
}
// GetProjectsMetaInfo retrieves meta information for a specific WhiteSource product
// GetProjectsMetaInfo retrieves the registered projects for a specific WhiteSource product
func (s *System) GetProjectsMetaInfo(productToken string) ([]Project, error) {
wsResponse := struct {
ProjectVitals []Project `json:"projectVitals"`
@ -144,37 +139,25 @@ func (s *System) GetProjectsMetaInfo(productToken string) ([]Project, error) {
ProductToken: productToken,
}
respBody, err := s.sendRequest(req)
err := s.sendRequestAndDecodeJSON(req, &wsResponse)
if err != nil {
return nil, errors.Wrap(err, "WhiteSource request failed")
}
err = json.Unmarshal(respBody, &wsResponse)
if err != nil {
return nil, errors.Wrap(err, "failed to parse WhiteSource response")
}
return wsResponse.ProjectVitals, nil
}
// GetProjectToken returns the project token for a project with a given name
func (s *System) GetProjectToken(productToken, projectName string) (string, error) {
var token string
project, err := s.GetProjectByName(productToken, projectName)
if err != nil {
return "", err
}
// returns a nil token and no error if not found
if project != nil {
token = project.Token
}
return token, nil
return project.Token, nil
}
// GetProjectVitals returns project meta info given a project token
func (s *System) GetProjectVitals(projectToken string) (*Project, error) {
// GetProjectByToken returns project meta info given a project token
func (s *System) GetProjectByToken(projectToken string) (Project, error) {
wsResponse := struct {
ProjectVitals []Project `json:"projectVitals"`
}{
@ -186,51 +169,48 @@ func (s *System) GetProjectVitals(projectToken string) (*Project, error) {
ProjectToken: projectToken,
}
respBody, err := s.sendRequest(req)
err := s.sendRequestAndDecodeJSON(req, &wsResponse)
if err != nil {
return nil, errors.Wrap(err, "WhiteSource request failed")
return Project{}, errors.Wrap(err, "WhiteSource request failed")
}
err = json.Unmarshal(respBody, &wsResponse)
if err != nil {
return nil, errors.Wrap(err, "failed to parse WhiteSource response")
if len(wsResponse.ProjectVitals) == 0 {
return Project{}, errors.Wrapf(err, "no project with token '%s' found in WhiteSource", projectToken)
}
return &wsResponse.ProjectVitals[0], nil
return wsResponse.ProjectVitals[0], nil
}
// GetProjectByName returns the finds and returns a project by name
func (s *System) GetProjectByName(productToken, projectName string) (*Project, error) {
var project *Project
// GetProjectByName fetches all projects and returns the one matching the given projectName, or none, if not found
func (s *System) GetProjectByName(productToken, projectName string) (Project, error) {
projects, err := s.GetProjectsMetaInfo(productToken)
if err != nil {
return nil, errors.Wrap(err, "failed to retrieve WhiteSource project meta info")
return Project{}, errors.Wrap(err, "failed to retrieve WhiteSource project meta info")
}
for _, proj := range projects {
if projectName == proj.Name {
project = &proj
break
for _, project := range projects {
if projectName == project.Name {
return project, nil
}
}
// returns a nil project and no error if no project exists with projectName
return project, nil
// returns empty project and no error. The reason seems to be that it makes polling until the project exists easier.
return Project{}, nil
}
// GetProjectsByIDs: get all project tokens given a list of project ids
// GetProjectsByIDs retrieves all projects for the given productToken and filters them by the given project ids
func (s *System) GetProjectsByIDs(productToken string, projectIDs []int64) ([]Project, error) {
var projectsMatched []Project
projects, err := s.GetProjectsMetaInfo(productToken)
if err != nil {
return nil, errors.Wrap(err, "failed to retrieve WhiteSource project meta info")
}
var projectsMatched []Project
for _, project := range projects {
for _, projectID := range projectIDs {
if projectID == project.ID {
projectsMatched = append(projectsMatched, project)
break
}
}
}
@ -269,20 +249,16 @@ func (s *System) GetProductName(productToken string) (string, error) {
ProductToken: productToken,
}
respBody, err := s.sendRequest(req)
err := s.sendRequestAndDecodeJSON(req, &wsResponse)
if err != nil {
return "", errors.Wrap(err, "WhiteSource request failed")
}
err = json.Unmarshal(respBody, &wsResponse)
if err != nil {
return "", errors.Wrap(err, "failed to parse WhiteSource response")
if len(wsResponse.ProductTags) == 0 {
return "", nil // fmt.Errorf("no product with token '%s' found in WhiteSource", productToken)
}
if len(wsResponse.ProductTags) > 0 {
return wsResponse.ProductTags[0].Name, nil
}
return "", nil
return wsResponse.ProductTags[0].Name, nil
}
// GetProjectRiskReport
@ -302,7 +278,6 @@ func (s *System) GetProjectRiskReport(projectToken string) ([]byte, error) {
// GetProjectVulnerabilityReport
func (s *System) GetProjectVulnerabilityReport(projectToken string, format string) ([]byte, error) {
req := Request{
RequestType: "getProjectVulnerabilityReport",
ProjectToken: projectToken,
@ -317,50 +292,6 @@ func (s *System) GetProjectVulnerabilityReport(projectToken string, format strin
return respBody, nil
}
// GetOrganizationProductVitals
func (s *System) GetOrganizationProductVitals() ([]Product, error) {
wsResponse := struct {
ProductVitals []Product `json:"productVitals"`
}{
ProductVitals: []Product{},
}
req := Request{
RequestType: "getOrganizationProductVitals",
}
respBody, err := s.sendRequest(req)
if err != nil {
return nil, errors.Wrap(err, "WhiteSource request failed")
}
err = json.Unmarshal(respBody, &wsResponse)
if err != nil {
return nil, errors.Wrap(err, "failed to parse WhiteSource response")
}
return wsResponse.ProductVitals, nil
}
// GetProductByName
func (s *System) GetProductByName(productName string) (*Product, error) {
var product Product
products, err := s.GetOrganizationProductVitals()
if err != nil {
return nil, errors.Wrap(err, "failed to getOrganizationProductVitals")
}
for _, prod := range products {
if prod.Name == productName {
product = prod
}
}
// returns nil, nil if no product was found
return &product, nil
}
// GetProjectAlerts
func (s *System) GetProjectAlerts(projectToken string) ([]Alert, error) {
wsResponse := struct {
@ -374,16 +305,11 @@ func (s *System) GetProjectAlerts(projectToken string) ([]Alert, error) {
ProjectToken: projectToken,
}
respBody, err := s.sendRequest(req)
err := s.sendRequestAndDecodeJSON(req, &wsResponse)
if err != nil {
return nil, errors.Wrap(err, "WhiteSource request failed")
}
err = json.Unmarshal(respBody, &wsResponse)
if err != nil {
return nil, errors.Wrap(err, "failed to parse WhiteSource response")
}
return wsResponse.Alerts, nil
}
@ -400,26 +326,47 @@ func (s *System) GetProjectLibraryLocations(projectToken string) ([]Library, err
ProjectToken: projectToken,
}
respBody, err := s.sendRequest(req)
err := s.sendRequestAndDecodeJSON(req, &wsResponse)
if err != nil {
return nil, errors.Wrap(err, "WhiteSource request failed")
}
err = json.Unmarshal(respBody, &wsResponse)
if err != nil {
return nil, errors.Wrap(err, "failed to parse WhiteSource response")
}
return wsResponse.Libraries, nil
}
func (s *System) sendRequestAndDecodeJSON(req Request, result interface{}) error {
respBody, err := s.sendRequest(req)
if err != nil {
return errors.Wrap(err, "WhiteSource request failed")
}
log.Entry().Debugf("response: %v", string(respBody))
errorResponse := struct {
ErrorCode string `json:"errorCode"`
ErrorMessage string `json:"errorMessage"`
}{}
err = json.Unmarshal(respBody, &errorResponse)
if err == nil && errorResponse.ErrorCode != "" {
return fmt.Errorf("invalid request, error code %s, message '%s'",
errorResponse.ErrorCode, errorResponse.ErrorMessage)
}
err = json.Unmarshal(respBody, result)
if err != nil {
return errors.Wrap(err, "failed to parse WhiteSource response")
}
return nil
}
func (s *System) sendRequest(req Request) ([]byte, error) {
var responseBody []byte
if req.UserKey == "" {
req.UserKey = s.UserToken
req.UserKey = s.userToken
}
if req.OrgToken == "" {
req.OrgToken = s.OrgToken
req.OrgToken = s.orgToken
}
body, err := json.Marshal(req)
@ -427,11 +374,11 @@ func (s *System) sendRequest(req Request) ([]byte, error) {
return responseBody, errors.Wrap(err, "failed to create WhiteSource request")
}
log.Entry().Debug(string(body))
log.Entry().Debugf("request: %v", string(body))
headers := http.Header{}
headers.Add("Content-Type", "application/json")
response, err := s.HTTPClient.SendRequest(http.MethodPost, s.ServerURL, bytes.NewBuffer(body), headers, nil)
response, err := s.httpClient.SendRequest(http.MethodPost, s.serverURL, bytes.NewBuffer(body), headers, nil)
if err != nil {
return responseBody, errors.Wrap(err, "failed to send request to WhiteSource")
@ -441,5 +388,6 @@ func (s *System) sendRequest(req Request) ([]byte, error) {
if err != nil {
return responseBody, errors.Wrap(err, "failed to read WhiteSource response")
}
return responseBody, nil
}

86
pkg/whitesource/whitesource_test.go
View File

@ -46,7 +46,7 @@ func TestGetProductsMetaInfo(t *testing.T) {
expectedRequestBody := `{"requestType":"getOrganizationProductVitals","userKey":"test_user_token","orgToken":"test_org_token"}`
sys := System{ServerURL: "https://my.test.server", HTTPClient: &myTestClient, OrgToken: "test_org_token", UserToken: "test_user_token"}
sys := System{serverURL: "https://my.test.server", httpClient: &myTestClient, orgToken: "test_org_token", userToken: "test_user_token"}
products, err := sys.GetProductsMetaInfo()
requestBody, err := ioutil.ReadAll(myTestClient.requestBody)
@ -77,8 +77,8 @@ func TestGetMetaInfoForProduct(t *testing.T) {
}`,
}
sys := System{ServerURL: "https://my.test.server", HTTPClient: &myTestClient, OrgToken: "test_org_token", UserToken: "test_user_token"}
product, err := sys.GetMetaInfoForProduct("Test Product 2")
sys := System{serverURL: "https://my.test.server", httpClient: &myTestClient, orgToken: "test_org_token", userToken: "test_user_token"}
product, err := sys.GetProductByName("Test Product 2")
assert.NoError(t, err)
assert.Equal(t, product.Name, "Test Product 2")
@ -104,7 +104,7 @@ func TestGetProjectsMetaInfo(t *testing.T) {
expectedRequestBody := `{"requestType":"getProductProjectVitals","userKey":"test_user_token","productToken":"test_product_token","orgToken":"test_org_token"}`
sys := System{ServerURL: "https://my.test.server", HTTPClient: &myTestClient, OrgToken: "test_org_token", UserToken: "test_user_token"}
sys := System{serverURL: "https://my.test.server", httpClient: &myTestClient, orgToken: "test_org_token", userToken: "test_user_token"}
projects, err := sys.GetProjectsMetaInfo("test_product_token")
requestBody, err := ioutil.ReadAll(myTestClient.requestBody)
@ -145,19 +145,27 @@ func TestGetProjectToken(t *testing.T) {
}`,
}
sys := System{ServerURL: "https://my.test.server", HTTPClient: &myTestClient, OrgToken: "test_org_token", UserToken: "test_user_token"}
sys := System{serverURL: "https://my.test.server", httpClient: &myTestClient, orgToken: "test_org_token", userToken: "test_user_token"}
projectToken, err := sys.GetProjectToken("test_product_token", "Test Project1")
assert.NoError(t, err)
assert.Equal(t, "test_project_token1", projectToken)
t.Parallel()
projectToken, err = sys.GetProjectToken("test_product_token", "Test Project2")
assert.NoError(t, err)
assert.Equal(t, "test_project_token2", projectToken)
t.Run("find project 1", func(t *testing.T) {
projectToken, err := sys.GetProjectToken("test_product_token", "Test Project1")
assert.NoError(t, err)
assert.Equal(t, "test_project_token1", projectToken)
})
projectToken, err = sys.GetProjectToken("test_product_token", "Test Project3")
assert.NoError(t, err)
assert.Equal(t, "", projectToken)
t.Run("find project 2", func(t *testing.T) {
projectToken, err := sys.GetProjectToken("test_product_token", "Test Project2")
assert.NoError(t, err)
assert.Equal(t, "test_project_token2", projectToken)
})
t.Run("not finding project 3 is an error", func(t *testing.T) {
projectToken, err := sys.GetProjectToken("test_product_token", "Test Project3")
assert.NoError(t, err)
assert.Equal(t, "", projectToken)
})
}
func TestGetProjectTokens(t *testing.T) {
@ -184,7 +192,7 @@ func TestGetProjectTokens(t *testing.T) {
}`,
}
sys := System{ServerURL: "https://my.test.server", HTTPClient: &myTestClient, OrgToken: "test_org_token", UserToken: "test_user_token"}
sys := System{serverURL: "https://my.test.server", httpClient: &myTestClient, orgToken: "test_org_token", userToken: "test_user_token"}
projectTokens, err := sys.GetProjectTokens("test_product_token", []string{"Test Project1", "Test Project2"})
assert.NoError(t, err)
@ -207,9 +215,55 @@ func TestGetProductName(t *testing.T) {
}`,
}
sys := System{ServerURL: "https://my.test.server", HTTPClient: &myTestClient, OrgToken: "test_org_token", UserToken: "test_user_token"}
sys := System{serverURL: "https://my.test.server", httpClient: &myTestClient, orgToken: "test_org_token", userToken: "test_user_token"}
productName, err := sys.GetProductName("test_product_token")
assert.NoError(t, err)
assert.Equal(t, "Test Product", productName)
}
func TestGetProjectsByIDs(t *testing.T) {
responseBody :=
`{
"projectVitals":[
{
"id":1,
"name":"prj-1"
},
{
"id":2,
"name":"prj-2"
},
{
"id":3,
"name":"prj-3"
},
{
"id":4,
"name":"prj-4"
}
]
}`
t.Parallel()
t.Run("find projects by ids", func(t *testing.T) {
myTestClient := whitesourceMockClient{responseBody: responseBody}
sys := System{serverURL: "https://my.test.server", httpClient: &myTestClient, orgToken: "test_org_token", userToken: "test_user_token"}
projects, err := sys.GetProjectsByIDs("test_product_token", []int64{4, 2})
assert.NoError(t, err)
assert.Equal(t, []Project{{ID: 2, Name: "prj-2"}, {ID: 4, Name: "prj-4"}}, projects)
})
t.Run("find no projects by ids", func(t *testing.T) {
myTestClient := whitesourceMockClient{responseBody: responseBody}
sys := System{serverURL: "https://my.test.server", httpClient: &myTestClient, orgToken: "test_org_token", userToken: "test_user_token"}
projects, err := sys.GetProjectsByIDs("test_product_token", []int64{5})
assert.NoError(t, err)
assert.Equal(t, []Project(nil), projects)
})
}

270
resources/metadata/whitesource.yaml
View File

@ -20,116 +20,42 @@ metadata:
spec:
inputs:
params:
- name: buildTool
type: string
description: "Defines the tool which is used for building the artifact."
mandatory: true
scope:
- GENERAL
- PARAMETERS
- STAGES
- STEPS
resourceRef:
- name: commonPipelineEnvironment
param: buildTool
- name: buildDescriptorFile
type: string
description: Explicit path to the build descriptor file.
mandatory: false
description: "Explicit path to the build descriptor file."
scope:
- PARAMETERS
- STAGES
- STEPS
default: null
conditions:
- conditionRef: strings-equal
params:
- name: scanType
value: golang
- name: buildDescriptorFile
- name: versioningModel
type: string
description: Explicit path to the build descriptor file.
mandatory: false
scope:
- PARAMETERS
- STAGES
- STEPS
default: ./pom.xml
conditions:
- conditionRef: strings-equal
params:
- name: scanType
value: maven
- name: buildDescriptorFile
type: string
description: Explicit path to the build descriptor file.
mandatory: false
scope:
- PARAMETERS
- STAGES
- STEPS
default: null
conditions:
- conditionRef: strings-equal
params:
- name: scanType
value: mta
- name: buildDescriptorFile
type: string
description: Explicit path to the build descriptor file.
mandatory: false
scope:
- PARAMETERS
- STAGES
- STEPS
default: ./package.json
conditions:
- conditionRef: strings-equal
params:
- name: scanType
value: npm
- name: buildDescriptorFile
type: string
description: Explicit path to the build descriptor file.
mandatory: false
scope:
- PARAMETERS
- STAGES
- STEPS
default: ./setup.py
conditions:
- conditionRef: strings-equal
params:
- name: scanType
value: pip
- name: buildDescriptorFile
type: string
description: Explicit path to the build descriptor file.
mandatory: false
scope:
- PARAMETERS
- STAGES
- STEPS
default: ./build.sbt
conditions:
- conditionRef: strings-equal
params:
- name: scanType
value: sbt
- name: buildDescriptorFile
type: string
description: Explicit path to the build descriptor file.
mandatory: false
scope:
- PARAMETERS
- STAGES
- STEPS
default: ./dub.json
conditions:
- conditionRef: strings-equal
params:
- name: scanType
value: dub
- name: defaultVersioningModel
type: string
description: The default project versioning model used in case `projectVersion` parameter is empty for creating the version based on the build descriptor version to report results in Whitesource, can be one of `'major'`, `'major-minor'`, `'semantic'`, `'full'`
description: "The default project versioning model used in case `projectVersion` parameter is
empty for creating the version based on the build descriptor version to report results in
Whitesource, can be one of `'major'`, `'major-minor'`, `'semantic'`, `'full'`"
scope:
- PARAMETERS
- STAGES
- STEPS
- GENERAL
default: "major"
aliases:
- name: defaultVersioningModel
- name: createProductFromPipeline
type: bool
description: Whether to create the related WhiteSource product on the fly based on the supplied pipeline configuration.
mandatory: false
description: "Whether to create the related WhiteSource product on the fly based on the supplied pipeline
configuration."
scope:
- PARAMETERS
- STAGES
@ -137,8 +63,7 @@ spec:
default: true
- name: securityVulnerabilities
type: bool
description: Whether security compliance is considered and reported as part of the assessment.
mandatory: false
description: "Whether security compliance is considered and reported as part of the assessment."
scope:
- PARAMETERS
- STAGES
@ -146,8 +71,7 @@ spec:
default: true
- name: timeout
type: string
description: Timeout in seconds until a HTTP call is forcefully terminated.
mandatory: false
description: "Timeout in seconds until a HTTP call is forcefully terminated."
scope:
- PARAMETERS
- STAGES
@ -155,8 +79,7 @@ spec:
default: 0
- name: agentDownloadUrl
type: string
description: URL used to download the latest version of the WhiteSource Unified Agent.
mandatory: false
description: "URL used to download the latest version of the WhiteSource Unified Agent."
scope:
- PARAMETERS
- STAGES
@ -164,8 +87,7 @@ spec:
default: https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar
- name: configFilePath
type: string
description: Explicit path to the WhiteSource Unified Agent configuration file.
mandatory: false
description: "Explicit path to the WhiteSource Unified Agent configuration file."
scope:
- PARAMETERS
- STAGES
@ -173,8 +95,7 @@ spec:
default: ./wss-generated-file.config
- name: reportDirectoryName
type: string
description: Name of the directory to save vulnerability/risk reports to
mandatory: false
description: "Name of the directory to save vulnerability/risk reports to"
scope:
- PARAMETERS
- STAGES
@ -182,8 +103,8 @@ spec:
default: "whitesource-reports"
- name: aggregateVersionWideReport
type: bool
description: "This does not run a scan, instead just generated a report for all projects with projectVersion = config.ProductVersion"
mandatory: false
description: "This does not run a scan, instead just generated a report for all projects with
projectVersion = config.ProductVersion"
scope:
- PARAMETERS
- STAGES
@ -191,8 +112,7 @@ spec:
default: false
- name: vulnerabilityReportFormat
type: string
description: Format of the file the vulnerability report is written to.
mandatory: false
description: "Format of the file the vulnerability report is written to."
possibleValues: [xlsx, json, xml]
scope:
- PARAMETERS
@ -201,10 +121,8 @@ spec:
default: xlsx
- name: parallelLimit
type: string
description:
"Limit of parallel jobs being run at once in case of `scanType:
'mta'` based scenarios, defaults to `15`."
mandatory: false
description: 'Limit of parallel jobs being run at once in case of `scanType:
''mta''` based scenarios, defaults to `15`.'
scope:
- PARAMETERS
- STAGES
@ -212,8 +130,7 @@ spec:
default: 15
- name: reporting
type: bool
description: Whether assessment is being done at all, defaults to `true`.
mandatory: false
description: "Whether assessment is being done at all, defaults to `true`"
scope:
- PARAMETERS
- STAGES
@ -221,8 +138,7 @@ spec:
default: true
- name: serviceUrl
type: string
description: URL to the WhiteSource server API used for communication.
mandatory: false
description: "URL to the WhiteSource server API used for communication."
scope:
- GENERAL
- PARAMETERS
@ -231,22 +147,19 @@ spec:
default: https://saas.whitesourcesoftware.com/api
- name: buildDescriptorExcludeList
type: "[]string"
description: List of build descriptors and therefore modules to exclude from the scan and assessment activities.
mandatory: false
description: "List of build descriptors and therefore modules to exclude from the scan and assessment activities."
scope:
- PARAMETERS
- STAGES
- STEPS
default: []
- name: orgToken
type: string
description: WhiteSource token identifying your organization.
description: "WhiteSource token identifying your organization."
scope:
- GENERAL
- PARAMETERS
- STAGES
- STEPS
default: null
secret: true
mandatory: true
resourceRef:
@ -254,13 +167,12 @@ spec:
type: secret
- name: userToken
type: string
description: WhiteSource token identifying the user executing the scan
description: "WhiteSource token identifying the user executing the scan"
scope:
- GENERAL
- PARAMETERS
- STAGES
- STEPS
default: null
secret: true
mandatory: true
resourceRef:
@ -268,8 +180,7 @@ spec:
type: secret
- name: licensingVulnerabilities
type: bool
description: Whether license compliance is considered and reported as part of the assessment.
mandatory: false
description: "Whether license compliance is considered and reported as part of the assessment."
scope:
- PARAMETERS
- STAGES
@ -277,52 +188,49 @@ spec:
default: true
- name: agentFileName
type: string
description: Locally used name for the Unified Agent jar file after download.
mandatory: false
description: "Locally used name for the Unified Agent jar file after download."
scope:
- PARAMETERS
- STAGES
- STEPS
default: wss-unified-agent.jar
- name: emailAddressesOfInitialProductAdmins
type: string
description: The list of email addresses to assign as product admins for newly created WhiteSource products.
mandatory: false
type: "[]string"
description: "The list of email addresses to assign as product admins for newly created WhiteSource products."
scope:
- PARAMETERS
- STAGES
- STEPS
default: []
- name: productVersion
type: string
description: Version of the WhiteSource product to be created and used for results aggregation, usually determined automatically.
mandatory: false
description: "Version of the WhiteSource product to be created and used for results aggregation,
usually determined automatically."
scope:
- GENERAL
- PARAMETERS
- STAGES
- STEPS
default: null
resourceRef:
- name: commonPipelineEnvironment
param: artifactVersion
- name: jreDownloadUrl
type: string
description: URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent.
mandatory: false
description: "URL used for downloading the Java Runtime Environment (JRE) required to run the
WhiteSource Unified Agent."
scope:
- GENERAL
- PARAMETERS
- STAGES
- STEPS
default: null
- name: productName
type: string
description: Name of the WhiteSource product to be created and used for results aggregation.
description: "Name of the WhiteSource product to be created and used for results aggregation."
mandatory: true
scope:
- GENERAL
- PARAMETERS
- STAGES
- STEPS
default: null
- name: projectName
aliases:
- name: whitesourceProjectName
@ -332,21 +240,17 @@ spec:
- PARAMETERS
- STAGES
- STEPS
default: '{{list .GroupID .ArtifactID | join "-" | trimAll "-"}}'
- name: projectToken
type: string
description: Project token to execute scan on
mandatory: false
scope:
- GENERAL
- PARAMETERS
- STAGES
- STEPS
default: null
- name: vulnerabilityReportTitle
type: string
description: Title of vulnerability report written during the assessment phase.
mandatory: false
description: "Title of vulnerability report written during the assessment phase."
scope:
- PARAMETERS
- STAGES
@ -354,37 +258,32 @@ spec:
default: WhiteSource Security Vulnerability Report
- name: installCommand
type: string
description: Install command that can be used to populate the default docker image for some scenarios.
mandatory: false
description: "Install command that can be used to populate the default docker image for some scenarios."
scope:
- PARAMETERS
- STAGES
- STEPS
default: null
- name: scanType
type: string
description: Type of development stack used to implement the solution.
mandatory: false
description: "Type of development stack used to implement the solution."
scope:
- GENERAL
- PARAMETERS
- STAGES
- STEPS
default: null
possibleValues: ["golang", "npm", "gradle", "pip"]
possibleValues: ["golang", "gradle", "maven", "mta", "npm", "pip", "yarn"]
- name: cvssSeverityLimit
type: string
description: Limit of tollerable CVSS v3 score upon assessment and in consequence fails the build, defaults to `-1`.
mandatory: false
description: "Limit of tolerable CVSS v3 score upon assessment and in consequence fails the build,
defaults to `-1`."
scope:
- PARAMETERS
- STAGES
- STEPS
default: -1
default: "-1"
- name: includes
type: string
description: Space separated list of file path patterns to include in the scan, slashes must be escaped for sed
mandatory: false
description: "Space separated list of file path patterns to include in the scan, slashes must be escaped for sed."
scope:
- PARAMETERS
- STAGES
@ -393,7 +292,6 @@ spec:
- name: excludes
type: string
description: Space separated list of file path patterns to exclude in the scan
mandatory: false
scope:
- PARAMETERS
- STAGES
@ -401,23 +299,65 @@ spec:
default: "tests/**/*.py **/src/test/**/*.java"
- name: productToken
type: string
description: Token of the WhiteSource product to be created and used for results aggregation, usually determined automatically.
mandatory: false
description: "Token of the WhiteSource product to be created and used for results aggregation,
usually determined automatically."
scope:
- GENERAL
- PARAMETERS
- STAGES
- STEPS
default: null
- name: agentParameters
type: string
description: Additional parameters passed to the Unified Agent command line.
mandatory: false
description: "Additional parameters passed to the Unified Agent command line."
scope:
- PARAMETERS
- STAGES
- STEPS
default: ""
# Global maven settings, should be added to all maven steps
- name: projectSettingsFile
type: string
description: "Path to the mvn settings file that should be used as project settings file."
scope:
- GENERAL
- STEPS
- STAGES
- PARAMETERS
aliases:
- name: maven/projectSettingsFile
- name: globalSettingsFile
type: string
description: "Path to the mvn settings file that should be used as global settings file."
scope:
- GENERAL
- STEPS
- STAGES
- PARAMETERS
aliases:
- name: maven/globalSettingsFile
- name: m2Path
type: string
description: "Path to the location of the local repository that should be used."
scope:
- GENERAL
- STEPS
- STAGES
- PARAMETERS
aliases:
- name: maven/m2Path
# Global npm settings, should be added to all npm steps
- name: defaultNpmRegistry
type: string
description: "URL of the npm registry to use. Defaults to https://registry.npmjs.org/"
scope:
- PARAMETERS
- GENERAL
- STAGES
- STEPS
aliases:
- name: npm/defaultNpmRegistry
secrets:
- name: userTokenCredentialsId
type: jenkins