package cmd import ( "archive/zip" "fmt" "io" "io/ioutil" "os" "path/filepath" "regexp" "sort" "strconv" "strings" "time" "encoding/xml" "github.com/SAP/jenkins-library/pkg/checkmarx" piperHttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/telemetry" "github.com/bmatcuk/doublestar" ) func checkmarxExecuteScan(config checkmarxExecuteScanOptions, telemetryData *telemetry.CustomData, influx *checkmarxExecuteScanInflux) error { client := &piperHttp.Client{} sys, err := checkmarx.NewSystemInstance(client, config.ServerURL, config.Username, config.Password) if err != nil { log.Entry().WithError(err).Fatalf("Failed to create Checkmarx client talking to URL %v", config.ServerURL) } runScan(config, sys, "./", influx) return nil } func runScan(config checkmarxExecuteScanOptions, sys checkmarx.System, workspace string, influx *checkmarxExecuteScanInflux) { team := loadTeam(sys, config.TeamName, config.TeamID) projectName := config.ProjectName project := loadExistingProject(sys, config.ProjectName, config.PullRequestName, team.ID) if project.Name == projectName { log.Entry().Infof("Project %v exists...", projectName) if len(config.Preset) > 0 { setPresetForProject(sys, project.ID, projectName, config.Preset, config.SourceEncoding) } } else { log.Entry().Infof("Project %v does not exist, starting to create it...", projectName) project = createAndConfigureNewProject(sys, projectName, team.ID, config.Preset, config.SourceEncoding) } uploadAndScan(config, sys, project, workspace, influx) } func loadTeam(sys checkmarx.System, teamName, teamID string) checkmarx.Team { teams := sys.GetTeams() team := checkmarx.Team{} if len(teams) > 0 { if len(teamName) > 0 { team = sys.FilterTeamByName(teams, teamName) } else { team = sys.FilterTeamByID(teams, teamID) } } if len(team.ID) == 0 { log.Entry().Fatalf("Failed to identify team by teamName %v as well as by checkmarxGroupId %v", teamName, teamID) } return team } func loadExistingProject(sys checkmarx.System, initialProjectName, pullRequestName, teamID string) checkmarx.Project { var project checkmarx.Project projectName := initialProjectName if len(pullRequestName) > 0 { projectName = fmt.Sprintf("%v_%v", initialProjectName, pullRequestName) projects := sys.GetProjectsByNameAndTeam(projectName, teamID) if len(projects) == 0 { projects = sys.GetProjectsByNameAndTeam(initialProjectName, teamID) if len(projects) > 0 { ok, branchProject := sys.GetProjectByID(sys.CreateBranch(projects[0].ID, projectName)) if !ok { log.Entry().Fatalf("Failed to create branch %v for project %v", projectName, initialProjectName) } project = branchProject } } } else { projects := sys.GetProjectsByNameAndTeam(projectName, teamID) if len(projects) > 0 { project = projects[0] log.Entry().Debugf("Loaded project with name %v", project.Name) } } return project } func zipWorkspaceFiles(workspace, filterPattern string) *os.File { zipFileName := filepath.Join(workspace, "workspace.zip") patterns := strings.Split(strings.ReplaceAll(strings.ReplaceAll(filterPattern, ", ", ","), " ,", ","), ",") sort.Strings(patterns) zipFile, err := os.Create(zipFileName) if err != nil { log.Entry().WithError(err).Fatal("Failed to create archive of project sources") } defer zipFile.Close() zipFolder(workspace, zipFile, patterns) return zipFile } func uploadAndScan(config checkmarxExecuteScanOptions, sys checkmarx.System, project checkmarx.Project, workspace string, influx *checkmarxExecuteScanInflux) { zipFile := zipWorkspaceFiles(workspace, config.FilterPattern) sourceCodeUploaded := sys.UploadProjectSourceCode(project.ID, zipFile.Name()) if sourceCodeUploaded { log.Entry().Debugf("Source code uploaded for project %v", project.Name) err := os.Remove(zipFile.Name()) if err != nil { log.Entry().WithError(err).Warnf("Failed to delete zipped source code for project %v", project.Name) } incremental := config.Incremental fullScanCycle, err := strconv.Atoi(config.FullScanCycle) if err != nil { log.Entry().WithError(err).Fatalf("Invalid configuration value for fullScanCycle %v, must be a positive int", config.FullScanCycle) } if incremental && config.FullScansScheduled && fullScanCycle > 0 && (getNumCoherentIncrementalScans(sys, project.ID)+1)%fullScanCycle == 0 { incremental = false } triggerScan(config, sys, project, workspace, incremental, influx) } else { log.Entry().Fatalf("Cannot upload source code for project %v", project.Name) } } func triggerScan(config checkmarxExecuteScanOptions, sys checkmarx.System, project checkmarx.Project, workspace string, incremental bool, influx *checkmarxExecuteScanInflux) { projectIsScanning, scan := sys.ScanProject(project.ID, incremental, false, !config.AvoidDuplicateProjectScans) if projectIsScanning { log.Entry().Debugf("Scanning project %v ", project.Name) pollScanStatus(sys, scan) log.Entry().Debugln("Scan finished") var reports []piperutils.Path if config.GeneratePdfReport { pdfReportName := createReportName(workspace, "CxSASTReport_%v.pdf") ok := downloadAndSaveReport(sys, pdfReportName, scan) if ok { reports = append(reports, piperutils.Path{Target: pdfReportName, Mandatory: true}) } } else { log.Entry().Debug("Report generation is disabled via configuration") } xmlReportName := createReportName(workspace, "CxSASTResults_%v.xml") results := getDetailedResults(sys, xmlReportName, scan.ID) reports = append(reports, piperutils.Path{Target: xmlReportName}) links := []piperutils.Path{piperutils.Path{Target: results["DeepLink"].(string), Name: "Checkmarx Web UI"}} piperutils.PersistReportsAndLinks("checkmarxExecuteScan", workspace, reports, links) reportToInflux(results, influx) insecure := false if config.VulnerabilityThresholdEnabled { insecure = enforceThresholds(config, results) } if insecure { if config.VulnerabilityThresholdResult == "FAILURE" { log.Entry().Fatalln("Checkmarx scan failed, the project is not compliant. For details see the archived report.") } log.Entry().Errorf("Checkmarx scan result set to %v, some results are not meeting defined thresholds. For details see the archived report.", config.VulnerabilityThresholdResult) } else { log.Entry().Infoln("Checkmarx scan finished") } } else { log.Entry().Fatalf("Cannot scan project %v", project.Name) } } func createReportName(workspace, reportFileNameTemplate string) string { regExpFileName := regexp.MustCompile(`[^\w\d]`) timeStamp, _ := time.Now().Local().MarshalText() return filepath.Join(workspace, fmt.Sprintf(reportFileNameTemplate, regExpFileName.ReplaceAllString(string(timeStamp), "_"))) } func pollScanStatus(sys checkmarx.System, scan checkmarx.Scan) { status := "Scan phase: New" pastStatus := status log.Entry().Info(status) for true { stepDetail := "..." stageDetail := "..." var detail checkmarx.ScanStatusDetail status, detail = sys.GetScanStatusAndDetail(scan.ID) if status == "Finished" || status == "Canceled" || status == "Failed" { break } if len(detail.Stage) > 0 { stageDetail = detail.Stage } if len(detail.Step) > 0 { stepDetail = detail.Step } status = fmt.Sprintf("Scan phase: %v (%v / %v)", status, stageDetail, stepDetail) if pastStatus != status { log.Entry().Info(status) pastStatus = status } log.Entry().Debug("Polling for status: sleeping...") time.Sleep(10 * time.Second) } if status == "Canceled" { log.Entry().Fatalln("Scan canceled via web interface") } if status == "Failed" { log.Entry().Fatalln("Scan failed, please check the Checkmarx UI for details") } } func reportToInflux(results map[string]interface{}, influx *checkmarxExecuteScanInflux) { influx.checkmarx_data.fields.high_issues = strconv.Itoa(results["High"].(map[string]int)["Issues"]) influx.checkmarx_data.fields.high_not_false_postive = strconv.Itoa(results["High"].(map[string]int)["NotFalsePositive"]) influx.checkmarx_data.fields.high_not_exploitable = strconv.Itoa(results["High"].(map[string]int)["NotExploitable"]) influx.checkmarx_data.fields.high_confirmed = strconv.Itoa(results["High"].(map[string]int)["Confirmed"]) influx.checkmarx_data.fields.high_urgent = strconv.Itoa(results["High"].(map[string]int)["Urgent"]) influx.checkmarx_data.fields.high_proposed_not_exploitable = strconv.Itoa(results["High"].(map[string]int)["ProposedNotExploitable"]) influx.checkmarx_data.fields.high_to_verify = strconv.Itoa(results["High"].(map[string]int)["ToVerify"]) influx.checkmarx_data.fields.medium_issues = strconv.Itoa(results["Medium"].(map[string]int)["Issues"]) influx.checkmarx_data.fields.medium_not_false_postive = strconv.Itoa(results["Medium"].(map[string]int)["NotFalsePositive"]) influx.checkmarx_data.fields.medium_not_exploitable = strconv.Itoa(results["Medium"].(map[string]int)["NotExploitable"]) influx.checkmarx_data.fields.medium_confirmed = strconv.Itoa(results["Medium"].(map[string]int)["Confirmed"]) influx.checkmarx_data.fields.medium_urgent = strconv.Itoa(results["Medium"].(map[string]int)["Urgent"]) influx.checkmarx_data.fields.medium_proposed_not_exploitable = strconv.Itoa(results["Medium"].(map[string]int)["ProposedNotExploitable"]) influx.checkmarx_data.fields.medium_to_verify = strconv.Itoa(results["Medium"].(map[string]int)["ToVerify"]) influx.checkmarx_data.fields.low_issues = strconv.Itoa(results["Low"].(map[string]int)["Issues"]) influx.checkmarx_data.fields.low_not_false_postive = strconv.Itoa(results["Low"].(map[string]int)["NotFalsePositive"]) influx.checkmarx_data.fields.low_not_exploitable = strconv.Itoa(results["Low"].(map[string]int)["NotExploitable"]) influx.checkmarx_data.fields.low_confirmed = strconv.Itoa(results["Low"].(map[string]int)["Confirmed"]) influx.checkmarx_data.fields.low_urgent = strconv.Itoa(results["Low"].(map[string]int)["Urgent"]) influx.checkmarx_data.fields.low_proposed_not_exploitable = strconv.Itoa(results["Low"].(map[string]int)["ProposedNotExploitable"]) influx.checkmarx_data.fields.low_to_verify = strconv.Itoa(results["Low"].(map[string]int)["ToVerify"]) influx.checkmarx_data.fields.information_issues = strconv.Itoa(results["Information"].(map[string]int)["Issues"]) influx.checkmarx_data.fields.information_not_false_postive = strconv.Itoa(results["Information"].(map[string]int)["NotFalsePositive"]) influx.checkmarx_data.fields.information_not_exploitable = strconv.Itoa(results["Information"].(map[string]int)["NotExploitable"]) influx.checkmarx_data.fields.information_confirmed = strconv.Itoa(results["Information"].(map[string]int)["Confirmed"]) influx.checkmarx_data.fields.information_urgent = strconv.Itoa(results["Information"].(map[string]int)["Urgent"]) influx.checkmarx_data.fields.information_proposed_not_exploitable = strconv.Itoa(results["Information"].(map[string]int)["ProposedNotExploitable"]) influx.checkmarx_data.fields.information_to_verify = strconv.Itoa(results["Information"].(map[string]int)["ToVerify"]) influx.checkmarx_data.fields.initiator_name = results["InitiatorName"].(string) influx.checkmarx_data.fields.owner = results["Owner"].(string) influx.checkmarx_data.fields.scan_id = results["ScanId"].(string) influx.checkmarx_data.fields.project_id = results["ProjectId"].(string) influx.checkmarx_data.fields.project_name = results["ProjectName"].(string) influx.checkmarx_data.fields.team = results["Team"].(string) influx.checkmarx_data.fields.team_full_path_on_report_date = results["TeamFullPathOnReportDate"].(string) influx.checkmarx_data.fields.scan_start = results["ScanStart"].(string) influx.checkmarx_data.fields.scan_time = results["ScanTime"].(string) influx.checkmarx_data.fields.lines_of_code_scanned = results["LinesOfCodeScanned"].(string) influx.checkmarx_data.fields.files_scanned = results["FilesScanned"].(string) influx.checkmarx_data.fields.checkmarx_version = results["CheckmarxVersion"].(string) influx.checkmarx_data.fields.scan_type = results["ScanType"].(string) influx.checkmarx_data.fields.preset = results["Preset"].(string) influx.checkmarx_data.fields.deep_link = results["DeepLink"].(string) influx.checkmarx_data.fields.report_creation_time = results["ReportCreationTime"].(string) } func downloadAndSaveReport(sys checkmarx.System, reportFileName string, scan checkmarx.Scan) bool { ok, report := generateAndDownloadReport(sys, scan.ID, "PDF") if ok { log.Entry().Debugf("Saving report to file %v...", reportFileName) ioutil.WriteFile(reportFileName, report, 0700) return true } log.Entry().Debugf("Failed to fetch report %v from backend...", reportFileName) return false } func enforceThresholds(config checkmarxExecuteScanOptions, results map[string]interface{}) bool { insecure := false cxHighThreshold := config.VulnerabilityThresholdHigh cxMediumThreshold := config.VulnerabilityThresholdMedium cxLowThreshold := config.VulnerabilityThresholdLow highValue := results["High"].(map[string]int)["NotFalsePositive"] mediumValue := results["Medium"].(map[string]int)["NotFalsePositive"] lowValue := results["Low"].(map[string]int)["NotFalsePositive"] var unit string highViolation := "" mediumViolation := "" lowViolation := "" if config.VulnerabilityThresholdUnit == "percentage" { unit = "%" highAudited := results["High"].(map[string]int)["Issues"] - results["High"].(map[string]int)["NotFalsePositive"] highOverall := results["High"].(map[string]int)["Issues"] if highOverall == 0 { highAudited = 1 highOverall = 1 } mediumAudited := results["Medium"].(map[string]int)["Issues"] - results["Medium"].(map[string]int)["NotFalsePositive"] mediumOverall := results["Medium"].(map[string]int)["Issues"] if mediumOverall == 0 { mediumAudited = 1 mediumOverall = 1 } lowAudited := results["Low"].(map[string]int)["Confirmed"] + results["Low"].(map[string]int)["NotExploitable"] lowOverall := results["Low"].(map[string]int)["Issues"] if lowOverall == 0 { lowAudited = 1 lowOverall = 1 } highValue = int(float32(highAudited) / float32(highOverall) * 100.0) mediumValue = int(float32(mediumAudited) / float32(mediumOverall) * 100.0) lowValue = int(float32(lowAudited) / float32(lowOverall) * 100.0) if highValue < cxHighThreshold { insecure = true highViolation = fmt.Sprintf("<-- %v %v deviation", cxHighThreshold-highValue, unit) } if mediumValue < cxMediumThreshold { insecure = true mediumViolation = fmt.Sprintf("<-- %v %v deviation", cxMediumThreshold-mediumValue, unit) } if lowValue < cxLowThreshold { insecure = true lowViolation = fmt.Sprintf("<-- %v %v deviation", cxLowThreshold-lowValue, unit) } } if config.VulnerabilityThresholdUnit == "absolute" { unit = " findings" if highValue > cxHighThreshold { insecure = true highViolation = fmt.Sprintf("<-- %v%v deviation", highValue-cxHighThreshold, unit) } if mediumValue > cxMediumThreshold { insecure = true mediumViolation = fmt.Sprintf("<-- %v%v deviation", mediumValue-cxMediumThreshold, unit) } if lowValue > cxLowThreshold { insecure = true lowViolation = fmt.Sprintf("<-- %v%v deviation", lowValue-cxLowThreshold, unit) } } log.Entry().Infoln("") log.Entry().Infof("High %v%v %v", highValue, unit, highViolation) log.Entry().Infof("Medium %v%v %v", mediumValue, unit, mediumViolation) log.Entry().Infof("Low %v%v %v", lowValue, unit, lowViolation) log.Entry().Infoln("") return insecure } func createAndConfigureNewProject(sys checkmarx.System, projectName, teamID, presetValue, engineConfiguration string) checkmarx.Project { ok, projectCreateResult := sys.CreateProject(projectName, teamID) if ok { if len(presetValue) > 0 { setPresetForProject(sys, projectCreateResult.ID, projectName, presetValue, engineConfiguration) } else { log.Entry().Fatalf("Preset not specified, creation of project %v failed", projectName) } projects := sys.GetProjectsByNameAndTeam(projectName, teamID) if len(projects) > 0 { log.Entry().Debugf("New Project %v created", projectName) return projects[0] } log.Entry().Fatalf("Failed to load newly created project %v", projectName) } log.Entry().Fatalf("Cannot create project %v", projectName) return checkmarx.Project{} } // loadPreset finds a checkmarx.Preset that has either the ID or Name given by presetValue. // presetValue is not expected to be empty. func loadPreset(sys checkmarx.System, presetValue string) (bool, checkmarx.Preset) { presets := sys.GetPresets() var preset checkmarx.Preset presetID, err := strconv.Atoi(presetValue) var configuredPresetID int var configuredPresetName string if err != nil { preset = sys.FilterPresetByName(presets, presetValue) configuredPresetName = presetValue } else { preset = sys.FilterPresetByID(presets, presetID) configuredPresetID = presetID } if configuredPresetID > 0 && preset.ID == configuredPresetID || len(configuredPresetName) > 0 && preset.Name == configuredPresetName { log.Entry().Infof("Loaded preset %v", preset.Name) return true, preset } log.Entry().Infof("Preset '%s' not found. Available presets are:", presetValue) for _, prs := range presets { log.Entry().Infof("preset id: %v, name: '%v'", prs.ID, prs.Name) } return false, checkmarx.Preset{} } // setPresetForProject is only called when it has already been established that the preset needs to be set. // It will exit via the logging framework in case the preset could be found, or the project could not be updated. func setPresetForProject(sys checkmarx.System, projectID int, projectName, presetValue, engineConfiguration string) { ok, preset := loadPreset(sys, presetValue) if ok { configurationUpdated := sys.UpdateProjectConfiguration(projectID, preset.ID, engineConfiguration) if configurationUpdated { log.Entry().Debugf("Configuration of project %v updated", projectName) } else { log.Entry().Fatalf("Updating configuration of project %v failed", projectName) } } else { log.Entry().Fatalf("Preset %v not found, configuration of project %v failed", presetValue, projectName) } } func generateAndDownloadReport(sys checkmarx.System, scanID int, reportType string) (bool, []byte) { success, report := sys.RequestNewReport(scanID, reportType) if success { finalStatus := 1 for { finalStatus = sys.GetReportStatus(report.ReportID).Status.ID if finalStatus != 1 { break } time.Sleep(10 * time.Second) } if finalStatus == 2 { return sys.DownloadReport(report.ReportID) } } return false, []byte{} } func getNumCoherentIncrementalScans(sys checkmarx.System, projectID int) int { ok, scans := sys.GetScans(projectID) count := 0 if ok { for _, scan := range scans { if !scan.IsIncremental { break } count++ } } return count } func getDetailedResults(sys checkmarx.System, reportFileName string, scanID int) map[string]interface{} { resultMap := map[string]interface{}{} ok, data := generateAndDownloadReport(sys, scanID, "XML") if ok && len(data) > 0 { ioutil.WriteFile(reportFileName, data, 0700) var xmlResult checkmarx.DetailedResult err := xml.Unmarshal(data, &xmlResult) if err != nil { log.Entry().Fatalf("Failed to unmarshal XML report for scan %v: %s", scanID, err) } resultMap["InitiatorName"] = xmlResult.InitiatorName resultMap["Owner"] = xmlResult.Owner resultMap["ScanId"] = xmlResult.ScanID resultMap["ProjectId"] = xmlResult.ProjectID resultMap["ProjectName"] = xmlResult.ProjectName resultMap["Team"] = xmlResult.Team resultMap["TeamFullPathOnReportDate"] = xmlResult.TeamFullPathOnReportDate resultMap["ScanStart"] = xmlResult.ScanStart resultMap["ScanTime"] = xmlResult.ScanTime resultMap["LinesOfCodeScanned"] = xmlResult.LinesOfCodeScanned resultMap["FilesScanned"] = xmlResult.FilesScanned resultMap["CheckmarxVersion"] = xmlResult.CheckmarxVersion resultMap["ScanType"] = xmlResult.ScanType resultMap["Preset"] = xmlResult.Preset resultMap["DeepLink"] = xmlResult.DeepLink resultMap["ReportCreationTime"] = xmlResult.ReportCreationTime resultMap["High"] = map[string]int{} resultMap["Medium"] = map[string]int{} resultMap["Low"] = map[string]int{} resultMap["Information"] = map[string]int{} for _, query := range xmlResult.Queries { for _, result := range query.Results { key := result.Severity var submap map[string]int if resultMap[key] == nil { submap = map[string]int{} resultMap[key] = submap } else { submap = resultMap[key].(map[string]int) } submap["Issues"]++ auditState := "ToVerify" switch result.State { case "1": auditState = "NotExploitable" break case "2": auditState = "Confirmed" break case "3": auditState = "Urgent" break case "4": auditState = "ProposedNotExploitable" break case "0": default: auditState = "ToVerify" break } submap[auditState]++ if result.FalsePositive != "True" { submap["NotFalsePositive"]++ } } } } return resultMap } func zipFolder(source string, zipFile io.Writer, patterns []string) error { archive := zip.NewWriter(zipFile) defer archive.Close() info, err := os.Stat(source) if err != nil { return nil } var baseDir string if info.IsDir() { baseDir = filepath.Base(source) } fileCount := 0 filepath.Walk(source, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if filterFileGlob(patterns, path, info) { return nil } header, err := zip.FileInfoHeader(info) if err != nil { return err } if baseDir != "" { header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source)) } if info.IsDir() { header.Name += "/" } else { header.Method = zip.Deflate } writer, err := archive.CreateHeader(header) if err != nil { return err } if info.IsDir() { return nil } file, err := os.Open(path) if err != nil { return err } defer file.Close() _, err = io.Copy(writer, file) fileCount++ return err }) log.Entry().Infof("Zipped %d files", fileCount) return err } func filterFileGlob(patterns []string, path string, info os.FileInfo) bool { for _, pattern := range patterns { negative := false if strings.HasPrefix(pattern, "!") { pattern = strings.TrimLeft(pattern, "!") negative = true } match, _ := doublestar.Match(pattern, path) if !info.IsDir() { if match && negative { return true } else if match && !negative { return false } } else { return false } } return true }