1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2024-12-12 10:55:20 +02:00
sap-jenkins-library/pkg/whitesource/scanNPM.go
Sven Merk b3f37650a2
SBOM creation for Mend (#3934)
* Fix docs and format

* Assessment format added

* Added sample file

* Added parsing

* Added packageurl implementation

* Slight refinement

* Refactored assessment options

* Adapted sample file

* First attempt of ws sbom gen

* Reworked SBOM generation

* Fix test code

* Add assessment handling

* Update dependencies

* Added golden test

* Small fix

Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2022-08-09 13:56:01 +02:00

231 lines
8.5 KiB
Go

package whitesource
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/pkg/errors"
)
const whiteSourceConfig = "whitesource.config.json"
func setValueAndLogChange(config map[string]interface{}, key string, value interface{}) {
oldValue, exists := config[key]
if exists && oldValue != value {
log.Entry().Infof("overwriting '%s' in %s: %v -> %v", key, whiteSourceConfig, oldValue, value)
}
config[key] = value
}
func setValueOmitIfPresent(config map[string]interface{}, key, omitIfPresent string, value interface{}) {
_, exists := config[omitIfPresent]
if exists {
return
}
setValueAndLogChange(config, key, value)
}
// writeWhitesourceConfigJSON creates or merges the file whitesource.config.json in the current
// directory from the given NPMScanOptions.
func (s *Scan) writeWhitesourceConfigJSON(config *ScanOptions, utils Utils, devDep, ignoreLsErrors bool) error {
var npmConfig = make(map[string]interface{})
exists, _ := utils.FileExists(whiteSourceConfig)
if exists {
fileContents, err := utils.FileRead(whiteSourceConfig)
if err != nil {
return fmt.Errorf("file '%s' already exists, but could not be read: %w", whiteSourceConfig, err)
}
err = json.Unmarshal(fileContents, &npmConfig)
if err != nil {
return fmt.Errorf("file '%s' already exists, but could not be parsed: %w", whiteSourceConfig, err)
}
log.Entry().Infof("The file '%s' already exists in the project. Changed config details will be logged.",
whiteSourceConfig)
}
npmConfig["apiKey"] = config.OrgToken
npmConfig["userKey"] = config.UserToken
setValueAndLogChange(npmConfig, "checkPolicies", true)
// When checkPolicies detects any violations, it will by default not update the WS project in the backend.
// Therefore we also need "forceUpdate".
setValueAndLogChange(npmConfig, "forceUpdate", true)
setValueAndLogChange(npmConfig, "productName", config.ProductName)
setValueAndLogChange(npmConfig, "productVer", s.ProductVersion)
setValueOmitIfPresent(npmConfig, "productToken", "projectToken", config.ProductToken)
if config.ProjectName != "" {
// In case there are other modules (i.e. maven modules in MTA projects),
// or more than one NPM module, setting the project name will lead to
// overwriting any previous scan results with the one from this module!
// If this is not provided, the WhiteSource project name will be generated
// from "name" in package.json plus " - " plus productVersion.
setValueAndLogChange(npmConfig, "projectName", config.ProjectName)
}
setValueAndLogChange(npmConfig, "devDep", devDep)
setValueAndLogChange(npmConfig, "ignoreNpmLsErrors", ignoreLsErrors)
jsonBuffer, err := json.Marshal(npmConfig)
if err != nil {
return fmt.Errorf("failed to generate '%s': %w", whiteSourceConfig, err)
}
err = utils.FileWrite(whiteSourceConfig, jsonBuffer, 0644)
if err != nil {
return fmt.Errorf("failed to write '%s': %w", whiteSourceConfig, err)
}
return nil
}
// ExecuteNpmScan iterates over all found npm modules and performs a scan in each one.
func (s *Scan) ExecuteNpmScan(config *ScanOptions, utils Utils) error {
s.AgentName = "WhiteSource NPM Plugin"
s.AgentVersion = "unknown"
modules, err := utils.FindPackageJSONFiles(config)
if err != nil {
return fmt.Errorf("failed to find package.json files with excludes: %w", err)
}
if len(modules) == 0 {
return fmt.Errorf("found no NPM modules to scan. Configured excludes: %v",
config.BuildDescriptorExcludeList)
}
for _, module := range modules {
err := s.executeNpmScanForModule(module, config, utils)
if err != nil {
return fmt.Errorf("failed to scan NPM module '%s': %w", module, err)
}
}
return nil
}
// executeNpmScanForModule generates a configuration file whitesource.config.json with appropriate values from config,
// installs all dependencies if necessary, and executes the scan via "npx whitesource run".
func (s *Scan) executeNpmScanForModule(modulePath string, config *ScanOptions, utils Utils) error {
log.Entry().Infof("Executing Whitesource scan for NPM module '%s'", modulePath)
resetDir, err := utils.Getwd()
if err != nil {
return fmt.Errorf("failed to obtain current directory: %w", err)
}
dir := filepath.Dir(modulePath)
if err := utils.Chdir(dir); err != nil {
return fmt.Errorf("failed to change into directory '%s': %w", dir, err)
}
defer func() {
err = utils.Chdir(resetDir)
if err != nil {
log.Entry().Errorf("Failed to reset into directory '%s': %v", resetDir, err)
}
}()
if err := s.writeWhitesourceConfigJSON(config, utils, false, true); err != nil {
return err
}
defer func() { _ = utils.FileRemove(whiteSourceConfig) }()
projectName, err := getNpmProjectName(modulePath, utils)
if err != nil {
return err
}
if err := reinstallNodeModulesIfLsFails(config, utils); err != nil {
return err
}
if err := s.AppendScannedProject(projectName); err != nil {
return err
}
return utils.RunExecutable("npx", "whitesource", "run")
}
// getNpmProjectName tries to read a property "name" of type string from the
// package.json file in the current directory and returns an error, if this is not possible.
func getNpmProjectName(modulePath string, utils Utils) (string, error) {
fileContents, err := utils.FileRead("package.json")
if err != nil {
return "", fmt.Errorf("could not read %s: %w", modulePath, err)
}
var packageJSON = make(map[string]interface{})
err = json.Unmarshal(fileContents, &packageJSON)
if err != nil {
return "", errors.Wrapf(err, "failed to unmarshall the file '%s'", modulePath)
}
projectNameEntry, exists := packageJSON["name"]
if !exists {
return "", fmt.Errorf("the file '%s' must configure a name", modulePath)
}
projectName, isString := projectNameEntry.(string)
if !isString {
return "", fmt.Errorf("the file '%s' must configure a name", modulePath)
}
return projectName, nil
}
// reinstallNodeModulesIfLsFails tests running of "npm ls".
// If that fails, the node_modules directory is cleared and the file "package-lock.json" is removed.
// Then "npm install" is performed. Without this, the npm whitesource plugin will consistently hang,
// when encountering npm ls errors, even with "ignoreNpmLsErrors:true" in the configuration.
// The consequence is that what was scanned is not guaranteed to be identical to what was built & deployed.
// This hack/work-around that should be removed once scanning it consistently performed using the Unified Agent.
// A possible reason for encountering "npm ls" errors in the first place is that a different node version
// is used for whitesourceExecuteScan due to a different docker image being used compared to the build stage.
func reinstallNodeModulesIfLsFails(config *ScanOptions, utils Utils) error {
// No need to have output from "npm ls" in the log
utils.Stdout(ioutil.Discard)
defer utils.Stdout(log.Writer())
err := utils.RunExecutable("npm", "ls")
if err == nil {
return nil
}
log.Entry().Warnf("'npm ls' failed. Re-installing NPM Node Modules")
err = utils.RemoveAll("node_modules")
if err != nil {
return fmt.Errorf("failed to remove node_modules directory: %w", err)
}
err = utils.MkdirAll("node_modules", os.ModePerm)
if err != nil {
return fmt.Errorf("failed to recreate node_modules directory: %w", err)
}
exists, _ := utils.FileExists("package-lock.json")
if exists {
err = utils.FileRemove("package-lock.json")
if err != nil {
return fmt.Errorf("failed to remove package-lock.json: %w", err)
}
}
// Passing only "package.json", because we are already inside the module's directory.
return utils.InstallAllNPMDependencies(config, []string{"package.json"})
}
// ExecuteYarnScan generates a configuration file whitesource.config.json with appropriate values from config,
// installs whitesource yarn plugin and executes the scan.
func (s *Scan) ExecuteYarnScan(config *ScanOptions, utils Utils) error {
// To stay compatible with what the step was doing before, trigger aggregation, although
// there is a great chance that it doesn't work with yarn the same way it doesn't with npm.
// Maybe the yarn code-path should be removed, and only npm stays.
config.ProjectName = s.AggregateProjectName
if err := s.writeWhitesourceConfigJSON(config, utils, true, false); err != nil {
return err
}
defer func() { _ = utils.FileRemove(whiteSourceConfig) }()
if err := utils.RunExecutable("yarn", "global", "add", "whitesource"); err != nil {
return err
}
if err := utils.RunExecutable("yarn", "install"); err != nil {
return err
}
if err := utils.RunExecutable("whitesource", "yarn"); err != nil {
return err
}
return nil
}