package cmd import ( "encoding/json" "fmt" "io" "io/ioutil" "os" "path" "path/filepath" "regexp" "strings" "time" "github.com/SAP/jenkins-library/pkg/command" piperDocker "github.com/SAP/jenkins-library/pkg/docker" "github.com/SAP/jenkins-library/pkg/log" StepResults "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/protecode" "github.com/SAP/jenkins-library/pkg/telemetry" ) type protecodeData struct { Target string `json:"target,omitempty"` Mandatory bool `json:"mandatory,omitempty"` ProductID string `json:"productID,omitempty"` ServerURL string `json:"serverUrl,omitempty"` FailOnSevereVulnerabilities bool `json:"failOnSevereVulnerabilities,omitempty"` ExcludeCVEs string `json:"excludeCVEs,omitempty"` Count string `json:"count,omitempty"` Cvss2GreaterOrEqualSeven string `json:"cvss2GreaterOrEqualSeven,omitempty"` Cvss3GreaterOrEqualSeven string `json:"cvss3GreaterOrEqualSeven,omitempty"` ExcludedVulnerabilities string `json:"excludedVulnerabilities,omitempty"` TriagedVulnerabilities string `json:"triagedVulnerabilities,omitempty"` HistoricalVulnerabilities string `json:"historicalVulnerabilities,omitempty"` Vulnerabilities []protecode.Vuln `json:"Vulnerabilities,omitempty"` } var reportPath = "./" var cachePath = "./cache" var cacheProtecodeImagePath = "/protecode/Image" var cacheProtecodePath = "/protecode" func protecodeExecuteScan(config protecodeExecuteScanOptions, telemetryData *telemetry.CustomData, influx *protecodeExecuteScanInflux) error { c := command.Command{} // reroute command output to loging framework c.Stdout(log.Writer()) c.Stderr(log.Writer()) dClient := createDockerClient(&config) return runProtecodeScan(&config, influx, dClient) } func runProtecodeScan(config *protecodeExecuteScanOptions, influx *protecodeExecuteScanInflux, dClient piperDocker.Download) error { correctDockerConfigEnvVar() var fileName, filePath string //create client for sending api request log.Entry().Debug("Create protecode client") client := createClient(config) if len(config.FetchURL) <= 0 { log.Entry().Debugf("Get docker image: %v, %v, %v, %v", config.ScanImage, config.DockerRegistryURL, config.FilePath, config.IncludeLayers) fileName, filePath = getDockerImage(dClient, config) if len(config.FilePath) <= 0 { (*config).FilePath = filePath log.Entry().Debugf("Filepath for upload image: %v", config.FilePath) } } log.Entry().Debug("Execute protecode scan") parsedResult := executeProtecodeScan(client, config, fileName, writeReportToFile) log.Entry().Debug("Write influx data") setInfluxData(influx, parsedResult) defer os.Remove(config.FilePath) deletePath := filepath.Join(cachePath, cacheProtecodePath) err := os.RemoveAll(deletePath) if err != nil { log.Entry().Warnf("Error during cleanup folder %v", err) } return nil } // reused by cmd/sonarExecuteScan.go // TODO: extract to version utils func handleArtifactVersion(artifactVersion string) string { matches, _ := regexp.MatchString("([\\d\\.]){1,}-[\\d]{14}([\\Wa-z\\d]{41})?", artifactVersion) if matches { split := strings.SplitN(artifactVersion, ".", 2) log.Entry().WithField("old", artifactVersion).WithField("new", split[0]).Debug("Trimming version to major version digit.") return split[0] } return artifactVersion } func getDockerImage(dClient piperDocker.Download, config *protecodeExecuteScanOptions) (string, string) { cacheImagePath := filepath.Join(cachePath, cacheProtecodeImagePath) deletePath := filepath.Join(cachePath, cacheProtecodePath) err := os.RemoveAll(deletePath) os.Mkdir(cacheImagePath, 600) imageSource, err := dClient.GetImageSource() if err != nil { log.Entry().WithError(err).Fatal("Error during get docker image source") } image, err := dClient.DownloadImageToPath(imageSource, cacheImagePath) if err != nil { log.Entry().Fatalf("Error during get docker image: %v", err) } tarFile, fileName := tarImage(config) if tarFile != nil { defer tarFile.Close() err = dClient.TarImage(tarFile, image) if err != nil { log.Entry().WithError(err).Fatal("Error during tar the docker image") } } resultFilePath := config.FilePath if len(config.FilePath) <= 0 { resultFilePath = cachePath } return fileName, resultFilePath } func tarImage(config *protecodeExecuteScanOptions) (*os.File, string) { if !(filepath.Ext(config.ScanImage) == ".tar" || filepath.Ext(config.ScanImage) == ".tar.gz" || filepath.Ext(config.ScanImage) == ".tgz") { artifactVersion := handleArtifactVersion(config.ArtifactVersion) fileName := fmt.Sprintf("%v%v.tar", strings.ReplaceAll(config.ScanImage, "/", "_"), strings.ReplaceAll(artifactVersion, ":", "_")) tarFileName := filepath.Join(cachePath, fileName) tarFile, err := os.Create(tarFileName) if err != nil { log.Entry().WithError(err).Fatal("Error during create tar for the docker image") } if err := os.Chmod(tarFileName, 0644); err != nil { log.Entry().WithError(err).Fatal("Error during create tar for the docker image") } return tarFile, fileName } return nil, config.ScanImage } func executeProtecodeScan(client protecode.Protecode, config *protecodeExecuteScanOptions, fileName string, writeReportToFile func(resp io.ReadCloser, reportFileName string) error) map[string]int { var parsedResult map[string]int = make(map[string]int) //load existing product by filename log.Entry().Debugf("Load existing product Group:%v Reuse:%v", config.Group, config.ReuseExisting) productID := client.LoadExistingProduct(config.Group, config.ReuseExisting) // check if no existing is found or reuse existing is false productID = uploadScanOrDeclareFetch(*config, productID, client, fileName) if productID <= 0 { log.Entry().Fatalf("The product id is not valid (product id %v <= zero)", productID) } //pollForResult log.Entry().Debugf("Poll for scan result %v", productID) result := client.PollForResult(productID, config.TimeoutMinutes) jsonData, _ := json.Marshal(result) filePath := filepath.Join(reportPath, "protecodescan_vulns.json") ioutil.WriteFile(filePath, jsonData, 0644) //check if result is ok else notify if len(result.Result.Status) > 0 && result.Result.Status == "F" { log.Entry().Fatalf("Please check the log and protecode backend for more details. URL: %v/products/%v", config.ServerURL, productID) } //loadReport log.Entry().Debugf("Load report %v for %v", config.ReportFileName, productID) resp := client.LoadReport(config.ReportFileName, productID) //save report to filesystem err := writeReportToFile(*resp, config.ReportFileName) if err != nil { return parsedResult } //clean scan from server log.Entry().Debugf("Delete scan %v for %v", config.CleanupMode, productID) client.DeleteScan(config.CleanupMode, productID) //count vulnerabilities log.Entry().Debug("Parse scan reult") parsedResult, vulns := client.ParseResultForInflux(result.Result, config.ExcludeCVEs) log.Entry().Debug("Write report to filesystem") writeReportDataToJSONFile(config, parsedResult, productID, vulns, ioutil.WriteFile) // write reports JSON reports := []StepResults.Path{ {Target: config.ReportFileName, Mandatory: true}, {Target: "protecodeExecuteScan.json", Mandatory: true}, {Target: "protecodescan_vulns.json", Mandatory: true}, } // write links JSON links := []StepResults.Path{ {Name: "Protecode WebUI", Target: fmt.Sprintf("%s/products/%v/", config.ServerURL, productID)}, {Name: "Protecode Report", Target: path.Join("artifact", config.ReportFileName), Scope: "job"}, } StepResults.PersistReportsAndLinks("protecodeExecuteScan", "", reports, links) return parsedResult } func setInfluxData(influx *protecodeExecuteScanInflux, result map[string]int) { influx.protecode_data.fields.historical_vulnerabilities = fmt.Sprintf("%v", result["historical_vulnerabilities"]) influx.protecode_data.fields.triaged_vulnerabilities = fmt.Sprintf("%v", result["triaged_vulnerabilities"]) influx.protecode_data.fields.excluded_vulnerabilities = fmt.Sprintf("%v", result["excluded_vulnerabilities"]) influx.protecode_data.fields.minor_vulnerabilities = fmt.Sprintf("%v", result["minor_vulnerabilities"]) influx.protecode_data.fields.major_vulnerabilities = fmt.Sprintf("%v", result["major_vulnerabilities"]) influx.protecode_data.fields.vulnerabilities = fmt.Sprintf("%v", result["vulnerabilities"]) } func writeReportDataToJSONFile(config *protecodeExecuteScanOptions, result map[string]int, productID int, vulns []protecode.Vuln, writeToFile func(f string, d []byte, p os.FileMode) error) { protecodeData := protecodeData{} protecodeData.ServerURL = config.ServerURL protecodeData.FailOnSevereVulnerabilities = config.FailOnSevereVulnerabilities protecodeData.ExcludeCVEs = config.ExcludeCVEs protecodeData.Target = config.ReportFileName protecodeData.Mandatory = true protecodeData.ProductID = fmt.Sprintf("%v", productID) protecodeData.Count = fmt.Sprintf("%v", result["count"]) protecodeData.Cvss2GreaterOrEqualSeven = fmt.Sprintf("%v", result["cvss2GreaterOrEqualSeven"]) protecodeData.Cvss3GreaterOrEqualSeven = fmt.Sprintf("%v", result["cvss3GreaterOrEqualSeven"]) protecodeData.ExcludedVulnerabilities = fmt.Sprintf("%v", result["excluded_vulnerabilities"]) protecodeData.TriagedVulnerabilities = fmt.Sprintf("%v", result["triaged_vulnerabilities"]) protecodeData.HistoricalVulnerabilities = fmt.Sprintf("%v", result["historical_vulnerabilities"]) protecodeData.Vulnerabilities = vulns jsonData, _ := json.Marshal(protecodeData) log.Entry().Infof("Protecode scan info, %v of which %v had a CVSS v2 score >= 7.0 and %v had a CVSS v3 score >= 7.0.\n %v vulnerabilities were excluded via configuration (%v) and %v vulnerabilities were triaged via the webUI.\nIn addition %v historical vulnerabilities were spotted. \n\n Vulnerabilities: %v", protecodeData.Count, protecodeData.Cvss2GreaterOrEqualSeven, protecodeData.Cvss3GreaterOrEqualSeven, protecodeData.ExcludedVulnerabilities, protecodeData.ExcludeCVEs, protecodeData.TriagedVulnerabilities, protecodeData.HistoricalVulnerabilities, protecodeData.Vulnerabilities) filePath := filepath.Join(reportPath, "protecodeExecuteScan.json") writeToFile(filePath, jsonData, 0644) } func createClient(config *protecodeExecuteScanOptions) protecode.Protecode { var duration time.Duration = time.Duration(time.Minute * 1) if len(config.TimeoutMinutes) > 0 { dur, err := time.ParseDuration(fmt.Sprintf("%vm", config.TimeoutMinutes)) if err != nil { log.Entry().Warnf("Failed to parse timeout %v, switched back to default timeout %v minutes", config.TimeoutMinutes, duration) } else { duration = dur } } pc := protecode.Protecode{} protecodeOptions := protecode.Options{ ServerURL: config.ServerURL, Logger: log.Entry().WithField("package", "SAP/jenkins-library/pkg/protecode"), Duration: duration, Username: config.Username, Password: config.Password, } pc.SetOptions(protecodeOptions) return pc } func createDockerClient(config *protecodeExecuteScanOptions) piperDocker.Download { dClientOptions := piperDocker.ClientOptions{ImageName: config.ScanImage, RegistryURL: config.DockerRegistryURL, LocalPath: config.FilePath, IncludeLayers: config.IncludeLayers} dClient := &piperDocker.Client{} dClient.SetOptions(dClientOptions) return dClient } func uploadScanOrDeclareFetch(config protecodeExecuteScanOptions, productID int, client protecode.Protecode, fileName string) int { //check if the LoadExistingProduct) before returns an valid product id, than scip this if !hasExisting(productID, config.ReuseExisting) { if len(config.FetchURL) > 0 { log.Entry().Debugf("Declare fetch url %v", config.FetchURL) resultData := client.DeclareFetchURL(config.CleanupMode, config.Group, config.FetchURL) productID = resultData.Result.ProductID } else { log.Entry().Debugf("Upload file path: %v", config.FilePath) if len(config.FilePath) <= 0 { log.Entry().Fatalf("There is no file path configured for upload : %v", config.FilePath) } pathToFile := filepath.Join(config.FilePath, fileName) if !(fileExists(pathToFile)) { log.Entry().Fatalf("There is no file for upload: %v", pathToFile) } combinedFileName := fileName if len(config.PullRequestName) > 0 { combinedFileName = fmt.Sprintf("%v_%v", config.PullRequestName, fileName) } resultData := client.UploadScanFile(config.CleanupMode, config.Group, pathToFile, combinedFileName) productID = resultData.Result.ProductID } } return productID } func fileExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) { return false } return !info.IsDir() } func hasExisting(productID int, reuseExisting bool) bool { if (productID > 0) || reuseExisting { return true } return false } var writeReportToFile = func(resp io.ReadCloser, reportFileName string) error { filePath := filepath.Join(reportPath, reportFileName) f, err := os.Create(filePath) if err == nil { defer f.Close() _, err = io.Copy(f, resp) } return err } func correctDockerConfigEnvVar() { path := os.Getenv("DOCKER_CONFIG") if len(path) > 0 { path, _ := filepath.Abs(path) path = filepath.Dir(path) fmt.Println("DOCKER_CONFIG: use parent directory") os.Setenv("DOCKER_CONFIG", path) } }