mirror of
https://github.com/SAP/jenkins-library.git
synced 2024-12-12 10:55:20 +02:00
b3f37650a2
* Fix docs and format * Assessment format added * Added sample file * Added parsing * Added packageurl implementation * Slight refinement * Refactored assessment options * Adapted sample file * First attempt of ws sbom gen * Reworked SBOM generation * Fix test code * Add assessment handling * Update dependencies * Added golden test * Small fix Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
231 lines
8.5 KiB
Go
231 lines
8.5 KiB
Go
package whitesource
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"github.com/SAP/jenkins-library/pkg/log"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
const whiteSourceConfig = "whitesource.config.json"
|
|
|
|
func setValueAndLogChange(config map[string]interface{}, key string, value interface{}) {
|
|
oldValue, exists := config[key]
|
|
if exists && oldValue != value {
|
|
log.Entry().Infof("overwriting '%s' in %s: %v -> %v", key, whiteSourceConfig, oldValue, value)
|
|
}
|
|
config[key] = value
|
|
}
|
|
|
|
func setValueOmitIfPresent(config map[string]interface{}, key, omitIfPresent string, value interface{}) {
|
|
_, exists := config[omitIfPresent]
|
|
if exists {
|
|
return
|
|
}
|
|
setValueAndLogChange(config, key, value)
|
|
}
|
|
|
|
// writeWhitesourceConfigJSON creates or merges the file whitesource.config.json in the current
|
|
// directory from the given NPMScanOptions.
|
|
func (s *Scan) writeWhitesourceConfigJSON(config *ScanOptions, utils Utils, devDep, ignoreLsErrors bool) error {
|
|
var npmConfig = make(map[string]interface{})
|
|
|
|
exists, _ := utils.FileExists(whiteSourceConfig)
|
|
if exists {
|
|
fileContents, err := utils.FileRead(whiteSourceConfig)
|
|
if err != nil {
|
|
return fmt.Errorf("file '%s' already exists, but could not be read: %w", whiteSourceConfig, err)
|
|
}
|
|
err = json.Unmarshal(fileContents, &npmConfig)
|
|
if err != nil {
|
|
return fmt.Errorf("file '%s' already exists, but could not be parsed: %w", whiteSourceConfig, err)
|
|
}
|
|
log.Entry().Infof("The file '%s' already exists in the project. Changed config details will be logged.",
|
|
whiteSourceConfig)
|
|
}
|
|
|
|
npmConfig["apiKey"] = config.OrgToken
|
|
npmConfig["userKey"] = config.UserToken
|
|
setValueAndLogChange(npmConfig, "checkPolicies", true)
|
|
// When checkPolicies detects any violations, it will by default not update the WS project in the backend.
|
|
// Therefore we also need "forceUpdate".
|
|
setValueAndLogChange(npmConfig, "forceUpdate", true)
|
|
setValueAndLogChange(npmConfig, "productName", config.ProductName)
|
|
setValueAndLogChange(npmConfig, "productVer", s.ProductVersion)
|
|
setValueOmitIfPresent(npmConfig, "productToken", "projectToken", config.ProductToken)
|
|
if config.ProjectName != "" {
|
|
// In case there are other modules (i.e. maven modules in MTA projects),
|
|
// or more than one NPM module, setting the project name will lead to
|
|
// overwriting any previous scan results with the one from this module!
|
|
// If this is not provided, the WhiteSource project name will be generated
|
|
// from "name" in package.json plus " - " plus productVersion.
|
|
setValueAndLogChange(npmConfig, "projectName", config.ProjectName)
|
|
}
|
|
setValueAndLogChange(npmConfig, "devDep", devDep)
|
|
setValueAndLogChange(npmConfig, "ignoreNpmLsErrors", ignoreLsErrors)
|
|
|
|
jsonBuffer, err := json.Marshal(npmConfig)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to generate '%s': %w", whiteSourceConfig, err)
|
|
}
|
|
|
|
err = utils.FileWrite(whiteSourceConfig, jsonBuffer, 0644)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to write '%s': %w", whiteSourceConfig, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ExecuteNpmScan iterates over all found npm modules and performs a scan in each one.
|
|
func (s *Scan) ExecuteNpmScan(config *ScanOptions, utils Utils) error {
|
|
s.AgentName = "WhiteSource NPM Plugin"
|
|
s.AgentVersion = "unknown"
|
|
modules, err := utils.FindPackageJSONFiles(config)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to find package.json files with excludes: %w", err)
|
|
}
|
|
if len(modules) == 0 {
|
|
return fmt.Errorf("found no NPM modules to scan. Configured excludes: %v",
|
|
config.BuildDescriptorExcludeList)
|
|
}
|
|
for _, module := range modules {
|
|
err := s.executeNpmScanForModule(module, config, utils)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to scan NPM module '%s': %w", module, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// executeNpmScanForModule generates a configuration file whitesource.config.json with appropriate values from config,
|
|
// installs all dependencies if necessary, and executes the scan via "npx whitesource run".
|
|
func (s *Scan) executeNpmScanForModule(modulePath string, config *ScanOptions, utils Utils) error {
|
|
log.Entry().Infof("Executing Whitesource scan for NPM module '%s'", modulePath)
|
|
|
|
resetDir, err := utils.Getwd()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to obtain current directory: %w", err)
|
|
}
|
|
|
|
dir := filepath.Dir(modulePath)
|
|
if err := utils.Chdir(dir); err != nil {
|
|
return fmt.Errorf("failed to change into directory '%s': %w", dir, err)
|
|
}
|
|
defer func() {
|
|
err = utils.Chdir(resetDir)
|
|
if err != nil {
|
|
log.Entry().Errorf("Failed to reset into directory '%s': %v", resetDir, err)
|
|
}
|
|
}()
|
|
|
|
if err := s.writeWhitesourceConfigJSON(config, utils, false, true); err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = utils.FileRemove(whiteSourceConfig) }()
|
|
|
|
projectName, err := getNpmProjectName(modulePath, utils)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := reinstallNodeModulesIfLsFails(config, utils); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := s.AppendScannedProject(projectName); err != nil {
|
|
return err
|
|
}
|
|
|
|
return utils.RunExecutable("npx", "whitesource", "run")
|
|
}
|
|
|
|
// getNpmProjectName tries to read a property "name" of type string from the
|
|
// package.json file in the current directory and returns an error, if this is not possible.
|
|
func getNpmProjectName(modulePath string, utils Utils) (string, error) {
|
|
fileContents, err := utils.FileRead("package.json")
|
|
if err != nil {
|
|
return "", fmt.Errorf("could not read %s: %w", modulePath, err)
|
|
}
|
|
var packageJSON = make(map[string]interface{})
|
|
err = json.Unmarshal(fileContents, &packageJSON)
|
|
if err != nil {
|
|
return "", errors.Wrapf(err, "failed to unmarshall the file '%s'", modulePath)
|
|
}
|
|
|
|
projectNameEntry, exists := packageJSON["name"]
|
|
if !exists {
|
|
return "", fmt.Errorf("the file '%s' must configure a name", modulePath)
|
|
}
|
|
|
|
projectName, isString := projectNameEntry.(string)
|
|
if !isString {
|
|
return "", fmt.Errorf("the file '%s' must configure a name", modulePath)
|
|
}
|
|
|
|
return projectName, nil
|
|
}
|
|
|
|
// reinstallNodeModulesIfLsFails tests running of "npm ls".
|
|
// If that fails, the node_modules directory is cleared and the file "package-lock.json" is removed.
|
|
// Then "npm install" is performed. Without this, the npm whitesource plugin will consistently hang,
|
|
// when encountering npm ls errors, even with "ignoreNpmLsErrors:true" in the configuration.
|
|
// The consequence is that what was scanned is not guaranteed to be identical to what was built & deployed.
|
|
// This hack/work-around that should be removed once scanning it consistently performed using the Unified Agent.
|
|
// A possible reason for encountering "npm ls" errors in the first place is that a different node version
|
|
// is used for whitesourceExecuteScan due to a different docker image being used compared to the build stage.
|
|
func reinstallNodeModulesIfLsFails(config *ScanOptions, utils Utils) error {
|
|
// No need to have output from "npm ls" in the log
|
|
utils.Stdout(ioutil.Discard)
|
|
defer utils.Stdout(log.Writer())
|
|
|
|
err := utils.RunExecutable("npm", "ls")
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
log.Entry().Warnf("'npm ls' failed. Re-installing NPM Node Modules")
|
|
err = utils.RemoveAll("node_modules")
|
|
if err != nil {
|
|
return fmt.Errorf("failed to remove node_modules directory: %w", err)
|
|
}
|
|
err = utils.MkdirAll("node_modules", os.ModePerm)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to recreate node_modules directory: %w", err)
|
|
}
|
|
exists, _ := utils.FileExists("package-lock.json")
|
|
if exists {
|
|
err = utils.FileRemove("package-lock.json")
|
|
if err != nil {
|
|
return fmt.Errorf("failed to remove package-lock.json: %w", err)
|
|
}
|
|
}
|
|
// Passing only "package.json", because we are already inside the module's directory.
|
|
return utils.InstallAllNPMDependencies(config, []string{"package.json"})
|
|
}
|
|
|
|
// ExecuteYarnScan generates a configuration file whitesource.config.json with appropriate values from config,
|
|
// installs whitesource yarn plugin and executes the scan.
|
|
func (s *Scan) ExecuteYarnScan(config *ScanOptions, utils Utils) error {
|
|
// To stay compatible with what the step was doing before, trigger aggregation, although
|
|
// there is a great chance that it doesn't work with yarn the same way it doesn't with npm.
|
|
// Maybe the yarn code-path should be removed, and only npm stays.
|
|
config.ProjectName = s.AggregateProjectName
|
|
if err := s.writeWhitesourceConfigJSON(config, utils, true, false); err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = utils.FileRemove(whiteSourceConfig) }()
|
|
if err := utils.RunExecutable("yarn", "global", "add", "whitesource"); err != nil {
|
|
return err
|
|
}
|
|
if err := utils.RunExecutable("yarn", "install"); err != nil {
|
|
return err
|
|
}
|
|
if err := utils.RunExecutable("whitesource", "yarn"); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|