package whitesource import ( "bufio" "bytes" "encoding/json" "fmt" "io" "os" "path/filepath" "regexp" "sync" "github.com/SAP/jenkins-library/pkg/log" "github.com/pkg/errors" ) const jvmTarGz = "jvm.tar.gz" const jvmDir = "./jvm" const projectRegEx = `Project name: ([^,]*), URL: (.*)` // ExecuteUAScan executes a scan with the Whitesource Unified Agent. func (s *Scan) ExecuteUAScan(config *ScanOptions, utils Utils) error { if config.BuildTool != "mta" { return s.ExecuteUAScanInPath(config, utils, config.ScanPath) } log.Entry().Infof("Executing WhiteSource UA scan for MTA project") pomExists, _ := utils.FileExists("pom.xml") if pomExists { mavenConfig := *config mavenConfig.BuildTool = "maven" if err := s.ExecuteUAScanInPath(&mavenConfig, utils, config.ScanPath); err != nil { return errors.Wrap(err, "failed to run scan for maven modules of mta") } } else { if pomFiles, _ := utils.Glob("**/pom.xml"); len(pomFiles) > 0 { log.SetErrorCategory(log.ErrorCustom) return fmt.Errorf("mta project with java modules does not contain an aggregator pom.xml in the root - this is mandatory") } } packageJSONFiles, err := utils.FindPackageJSONFiles(config) if err != nil { return errors.Wrap(err, "failed to find package.json files") } if len(packageJSONFiles) > 0 { npmConfig := *config npmConfig.BuildTool = "npm" for _, packageJSONFile := range packageJSONFiles { // we only need the path here modulePath, _ := filepath.Split(packageJSONFile) projectName, err := getProjectNameFromPackageJSON(packageJSONFile, utils) if err != nil { return errors.Wrapf(err, "failed retrieve project name") } npmConfig.ProjectName = projectName // ToDo: likely needs to be refactored, AggregateProjectName should only be available if we want to force aggregation? s.AggregateProjectName = projectName if err := s.ExecuteUAScanInPath(&npmConfig, utils, modulePath); err != nil { return errors.Wrapf(err, "failed to run scan for npm module %v", modulePath) } } } _ = removeJre(filepath.Join(jvmDir, "bin", "java"), utils) return nil } // ExecuteUAScanInPath executes a scan with the Whitesource Unified Agent in a dedicated scanPath. func (s *Scan) ExecuteUAScanInPath(config *ScanOptions, utils Utils, scanPath string) error { // Download the unified agent jar file if one does not exist err := downloadAgent(config, utils) if err != nil { return err } // Download JRE in case none is available javaPath, err := downloadJre(config, utils) if err != nil { return err } // ToDo: Check if Download of Docker/container image should be done here instead of in cmd/whitesourceExecuteScan.go // ToDo: check if this is required if err := s.AppendScannedProject(s.AggregateProjectName); err != nil { return err } configPath, err := config.RewriteUAConfigurationFile(utils, s.AggregateProjectName) if err != nil { return err } if len(scanPath) == 0 { scanPath = "." } // log parsing in order to identify the projects WhiteSource really scanned // we may refactor this in case there is a safer way to identify the projects e.g. via REST API //ToDO: we only need stdOut or stdErr, let's see where UA writes to ... prOut, stdOut := io.Pipe() trOut := io.TeeReader(prOut, os.Stderr) utils.Stdout(stdOut) prErr, stdErr := io.Pipe() trErr := io.TeeReader(prErr, os.Stderr) utils.Stdout(stdErr) var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() scanLog(trOut, s) }() go func() { defer wg.Done() scanLog(trErr, s) }() err = utils.RunExecutable(javaPath, "-jar", config.AgentFileName, "-d", scanPath, "-c", configPath, "-wss.url", config.AgentURL) if err := removeJre(javaPath, utils); err != nil { log.Entry().Warning(err) } if err != nil { if err := removeJre(javaPath, utils); err != nil { log.Entry().Warning(err) } exitCode := utils.GetExitCode() log.Entry().Infof("WhiteSource scan failed with exit code %v", exitCode) evaluateExitCode(exitCode) return errors.Wrapf(err, "failed to execute WhiteSource scan with exit code %v", exitCode) } return nil } func evaluateExitCode(exitCode int) { switch exitCode { case 255: log.Entry().Info("General error has occurred.") log.SetErrorCategory(log.ErrorUndefined) case 254: log.Entry().Info("Whitesource found one or multiple policy violations.") log.SetErrorCategory(log.ErrorCompliance) case 253: log.Entry().Info("The local scan client failed to execute the scan.") log.SetErrorCategory(log.ErrorUndefined) case 252: log.Entry().Info("There was a failure in the connection to the WhiteSource servers.") log.SetErrorCategory(log.ErrorInfrastructure) case 251: log.Entry().Info("The server failed to analyze the scan.") log.SetErrorCategory(log.ErrorService) case 250: log.Entry().Info("One of the package manager's prerequisite steps (e.g. npm install) failed.") log.SetErrorCategory(log.ErrorCustom) default: log.Entry().Info("Whitesource scan failed with unknown error code") log.SetErrorCategory(log.ErrorUndefined) } } // downloadAgent downloads the unified agent jar file if one does not exist func downloadAgent(config *ScanOptions, utils Utils) error { agentFile := config.AgentFileName exists, err := utils.FileExists(agentFile) if err != nil { return errors.Wrapf(err, "failed to check if file '%s' exists", agentFile) } if !exists { err := utils.DownloadFile(config.AgentDownloadURL, agentFile, nil, nil) if err != nil { return errors.Wrapf(err, "failed to download unified agent from URL '%s' to file '%s'", config.AgentDownloadURL, agentFile) } } return nil } // downloadJre downloads the a JRE in case no java command can be executed func downloadJre(config *ScanOptions, utils Utils) (string, error) { // cater for multiple executions if exists, _ := utils.FileExists(filepath.Join(jvmDir, "bin", "java")); exists { return filepath.Join(jvmDir, "bin", "java"), nil } err := utils.RunExecutable("java", "-version") javaPath := "java" if err != nil { log.Entry().Infof("No Java installation found, downloading JVM from %v", config.JreDownloadURL) err := utils.DownloadFile(config.JreDownloadURL, jvmTarGz, nil, nil) if err != nil { return "", errors.Wrapf(err, "failed to download jre from URL '%s'", config.JreDownloadURL) } // ToDo: replace tar call with go library call err = utils.MkdirAll(jvmDir, 0755) err = utils.RunExecutable("tar", fmt.Sprintf("--directory=%v", jvmDir), "--strip-components=1", "-xzf", jvmTarGz) if err != nil { return "", errors.Wrapf(err, "failed to extract %v", jvmTarGz) } log.Entry().Info("Java successfully installed") javaPath = filepath.Join(jvmDir, "bin", "java") } return javaPath, nil } func removeJre(javaPath string, utils Utils) error { if javaPath == "java" { return nil } if err := utils.RemoveAll(jvmDir); err != nil { return fmt.Errorf("failed to remove downloaded and extracted jvm from %v", jvmDir) } log.Entry().Debugf("Java successfully removed from %v", jvmDir) if err := utils.FileRemove(jvmTarGz); err != nil { return fmt.Errorf("failed to remove downloaded %v", jvmTarGz) } log.Entry().Debugf("%v successfully removed", jvmTarGz) return nil } func getProjectNameFromPackageJSON(packageJSONPath string, utils Utils) (string, error) { fileContents, err := utils.FileRead(packageJSONPath) if err != nil { return "", errors.Wrapf(err, "failed to read file %v", packageJSONPath) } var packageJSON = make(map[string]interface{}) if err := json.Unmarshal(fileContents, &packageJSON); err != nil { return "", errors.Wrapf(err, "failed to read file content of %v", packageJSONPath) } projectNameEntry, exists := packageJSON["name"] if !exists { return "", fmt.Errorf("the file '%s' must configure a name", packageJSONPath) } projectName, isString := projectNameEntry.(string) if !isString { return "", fmt.Errorf("the file '%s' must configure a name as string", packageJSONPath) } return projectName, nil } func scanLog(in io.Reader, scan *Scan) { scanner := bufio.NewScanner(in) scanner.Split(scanShortLines) for scanner.Scan() { line := scanner.Text() parseForProjects(line, scan) } if err := scanner.Err(); err != nil { log.Entry().WithError(err).Info("failed to scan log file") } } func parseForProjects(logLine string, scan *Scan) { compile := regexp.MustCompile(projectRegEx) values := compile.FindStringSubmatch(logLine) if len(values) > 0 && scan.scannedProjects != nil && len(scan.scannedProjects[values[1]].Name) == 0 { scan.scannedProjects[values[1]] = Project{Name: values[1]} } } func scanShortLines(data []byte, atEOF bool) (advance int, token []byte, err error) { lenData := len(data) if atEOF && lenData == 0 { return 0, nil, nil } if lenData > 32767 && !bytes.Contains(data[0:lenData], []byte("\n")) { // we will neglect long output // no use cases known where this would be relevant return lenData, nil, nil } if i := bytes.IndexByte(data, '\n'); i >= 0 && i < 32767 { // We have a full newline-terminated line with a size limit // Size limit is required since otherwise scanner would stall return i + 1, data[0:i], nil } // If we're at EOF, we have a final, non-terminated line. Return it. if atEOF { return len(data), data, nil } // Request more data. return 0, nil, nil }