package protecode import ( "bytes" "encoding/json" "fmt" "io" "net/http" "net/url" "strconv" "strings" "time" piperHttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" "github.com/sirupsen/logrus" ) // ReportsDirectory defines the subfolder for the Protecode reports which are generated const ReportsDirectory = "protecode" // ProductData holds the product information of the protecode product type ProductData struct { Products []Product `json:"products,omitempty"` } // Product holds the id of the protecode product type Product struct { ProductID int `json:"product_id,omitempty"` FileName string `json:"name,omitempty"` } //ResultData holds the information about the protecode result type ResultData struct { Result Result `json:"results,omitempty"` } //Result holds the detail information about the protecode result type Result struct { ProductID int `json:"product_id,omitempty"` ReportURL string `json:"report_url,omitempty"` Status string `json:"status,omitempty"` Components []Component `json:"components,omitempty"` } //Component the protecode component information type Component struct { Vulns []Vulnerability `json:"vulns,omitempty"` } //Vulnerability the protecode vulnerability information type Vulnerability struct { Exact bool `json:"exact,omitempty"` Vuln Vuln `json:"vuln,omitempty"` Triage []Triage `json:"triage,omitempty"` } //Vuln holds the inforamtion about the vulnerability type Vuln struct { Cve string `json:"cve,omitempty"` Cvss float64 `json:"cvss,omitempty"` Cvss3Score string `json:"cvss3_score,omitempty"` } //Triage holds the triaging information type Triage struct { ID int `json:"id,omitempty"` VulnID string `json:"vuln_id,omitempty"` Component string `json:"component,omitempty"` Vendor string `json:"vendor,omitempty"` Codetype string `json:"codetype,omitempty"` Version string `json:"version,omitempty"` Modified string `json:"modified,omitempty"` Scope string `json:"scope,omitempty"` Description string `json:"description,omitempty"` User User `json:"user,omitempty"` } //User holds the user information type User struct { ID int `json:"id,omitempty"` Email string `json:"email,omitempty"` Girstname string `json:"firstname,omitempty"` Lastname string `json:"lastname,omitempty"` Username string `json:"username,omitempty"` } //Protecode ist the protecode client which is used by the step type Protecode struct { serverURL string client piperHttp.Uploader duration time.Duration logger *logrus.Entry } // Just calls SetOptions which makes sure logger is set. // Added to make test code more resilient func makeProtecode(opts Options) Protecode { ret := Protecode{} ret.SetOptions(opts) return ret } //Options struct which can be used to configure the Protecode struct type Options struct { ServerURL string Duration time.Duration Username string Password string Logger *logrus.Entry } //SetOptions setter function to set the internal properties of the protecode func (pc *Protecode) SetOptions(options Options) { pc.serverURL = options.ServerURL pc.client = &piperHttp.Client{} pc.duration = options.Duration if options.Logger != nil { pc.logger = options.Logger } else { pc.logger = log.Entry().WithField("package", "SAP/jenkins-library/pkg/protecode") } httpOptions := piperHttp.ClientOptions{MaxRequestDuration: options.Duration, Username: options.Username, Password: options.Password, Logger: options.Logger} pc.client.SetOptions(httpOptions) } //SetHttpClient setter function to set the http client func (pc *Protecode) SetHttpClient(client piperHttp.Uploader) { pc.client = client } func (pc *Protecode) createURL(path string, pValue string, fParam string) string { protecodeURL, err := url.Parse(pc.serverURL) if err != nil { //TODO: bubble up error pc.logger.WithError(err).Fatal("Malformed URL") } if len(path) > 0 { protecodeURL.Path += fmt.Sprintf("%v", path) } if len(pValue) > 0 { protecodeURL.Path += fmt.Sprintf("%v", pValue) } // Prepare Query Parameters if len(fParam) > 0 { // encodedFParam := url.QueryEscape(fParam) params := url.Values{} params.Add("q", fmt.Sprintf("file:%v", fParam)) // Add Query Parameters to the URL protecodeURL.RawQuery = params.Encode() // Escape Query Parameters } return protecodeURL.String() } func (pc *Protecode) mapResponse(r io.ReadCloser, response interface{}) { defer r.Close() buf := new(bytes.Buffer) buf.ReadFrom(r) newStr := buf.String() if len(newStr) > 0 { unquoted, err := strconv.Unquote(newStr) if err != nil { err = json.Unmarshal([]byte(newStr), response) if err != nil { //TODO: bubble up error pc.logger.WithError(err).Fatalf("Error during unqote response: %v", newStr) } } else { err = json.Unmarshal([]byte(unquoted), response) } if err != nil { //TODO: bubble up error pc.logger.WithError(err).Fatalf("Error during decode response: %v", newStr) } } } func (pc *Protecode) sendAPIRequest(method string, url string, headers map[string][]string) (*io.ReadCloser, int, error) { r, err := pc.client.SendRequest(method, url, nil, headers, nil) if err != nil { if r != nil { return nil, r.StatusCode, err } return nil, 400, err } //return &r.Body, nil return &r.Body, r.StatusCode, nil } // ParseResultForInflux parses the result from the scan into the internal format func (pc *Protecode) ParseResultForInflux(result Result, excludeCVEs string) (map[string]int, []Vuln) { var vulns []Vuln var m map[string]int = make(map[string]int) m["count"] = 0 m["cvss2GreaterOrEqualSeven"] = 0 m["cvss3GreaterOrEqualSeven"] = 0 m["historical_vulnerabilities"] = 0 m["triaged_vulnerabilities"] = 0 m["excluded_vulnerabilities"] = 0 m["minor_vulnerabilities"] = 0 m["major_vulnerabilities"] = 0 m["vulnerabilities"] = 0 for _, components := range result.Components { for _, vulnerability := range components.Vulns { exact := isExact(vulnerability) countVulnerability := isExact(vulnerability) && !isExcluded(vulnerability, excludeCVEs) && !isTriaged(vulnerability) if exact && isExcluded(vulnerability, excludeCVEs) { m["excluded_vulnerabilities"]++ } if exact && isTriaged(vulnerability) { m["triaged_vulnerabilities"]++ } if countVulnerability { m["count"]++ m["vulnerabilities"]++ //collect all vulns here vulns = append(vulns, vulnerability.Vuln) } if countVulnerability && isSevereCVSS3(vulnerability) { m["cvss3GreaterOrEqualSeven"]++ m["major_vulnerabilities"]++ } if countVulnerability && isSevereCVSS2(vulnerability) { m["cvss2GreaterOrEqualSeven"]++ m["major_vulnerabilities"]++ } if countVulnerability && !isSevereCVSS3(vulnerability) && !isSevereCVSS2(vulnerability) { m["minor_vulnerabilities"]++ } if !exact { m["historical_vulnerabilities"]++ } } } return m, vulns } func isExact(vulnerability Vulnerability) bool { return vulnerability.Exact } func isExcluded(vulnerability Vulnerability, excludeCVEs string) bool { return strings.Contains(excludeCVEs, vulnerability.Vuln.Cve) } func isTriaged(vulnerability Vulnerability) bool { return len(vulnerability.Triage) > 0 } func isSevereCVSS3(vulnerability Vulnerability) bool { threshold := 7.0 cvss3, _ := strconv.ParseFloat(vulnerability.Vuln.Cvss3Score, 64) return cvss3 >= threshold } func isSevereCVSS2(vulnerability Vulnerability) bool { threshold := 7.0 cvss3, _ := strconv.ParseFloat(vulnerability.Vuln.Cvss3Score, 64) return cvss3 == 0 && vulnerability.Vuln.Cvss >= threshold } // DeleteScan deletes if configured the scan on the protecode server func (pc *Protecode) DeleteScan(cleanupMode string, productID int) { switch cleanupMode { case "none": case "binary": case "complete": pc.logger.Info("Deleting scan from server.") protecodeURL := pc.createURL("/api/product/", fmt.Sprintf("%v/", productID), "") headers := map[string][]string{} pc.sendAPIRequest("DELETE", protecodeURL, headers) default: //TODO: bubble up error pc.logger.Fatalf("Unknown cleanup mode %v", cleanupMode) } } // LoadReport loads the report of the protecode scan func (pc *Protecode) LoadReport(reportFileName string, productID int) *io.ReadCloser { protecodeURL := pc.createURL("/api/product/", fmt.Sprintf("%v/pdf-report", productID), "") headers := map[string][]string{ "Cache-Control": {"no-cache, no-store, must-revalidate"}, "Pragma": {"no-cache"}, "Outputfile": {reportFileName}, } readCloser, _, err := pc.sendAPIRequest(http.MethodGet, protecodeURL, headers) if err != nil { //TODO: bubble up error pc.logger.WithError(err).Fatalf("It is not possible to load report %v", protecodeURL) } return readCloser } // UploadScanFile upload the scan file to the protecode server func (pc *Protecode) UploadScanFile(cleanupMode, group, filePath, fileName, version string, productID int, replaceBinary bool) *ResultData { log.Entry().Debugf("[DEBUG] ===> UploadScanFile started.....") deleteBinary := (cleanupMode == "binary" || cleanupMode == "complete") var headers = make(map[string][]string) if (replaceBinary) && (version != "") { log.Entry().Debugf("[DEBUG] ===> replaceBinary && version != empty ") headers = map[string][]string{"Group": {group}, "Delete-Binary": {fmt.Sprintf("%v", deleteBinary)}, "Replace": {fmt.Sprintf("%v", productID)}, "Version": {version}} } else if replaceBinary { log.Entry().Debugf("[DEBUG] ===> replaceBinary") headers = map[string][]string{"Group": {group}, "Delete-Binary": {fmt.Sprintf("%v", deleteBinary)}, "Replace": {fmt.Sprintf("%v", productID)}} } else if version != "" { log.Entry().Debugf("[DEBUG] ===> version != empty ") headers = map[string][]string{"Group": {group}, "Delete-Binary": {fmt.Sprintf("%v", deleteBinary)}, "Version": {version}} } else { log.Entry().Debugf("[DEBUG] ===> replaceBinary is false and version == empty") headers = map[string][]string{"Group": {group}, "Delete-Binary": {fmt.Sprintf("%v", deleteBinary)}} } // log.Entry().Debugf("[DEBUG] ===> Headers for UploadScanFile upload: %v", headers) uploadURL := fmt.Sprintf("%v/api/upload/%v", pc.serverURL, fileName) r, err := pc.client.UploadRequest(http.MethodPut, uploadURL, filePath, "file", headers, nil, "binary") if err != nil { //TODO: bubble up error pc.logger.WithError(err).Fatalf("Error during upload request %v", uploadURL) } else { pc.logger.Info("Upload successful") } // log.Entry().Debugf("[DEBUG] ===> Upload request r: %v", r) // log.Entry().Debugf("[DEBUG] ===> Upload request r.StatusCode: %v", r.StatusCode) // For replaceBinary option response doesn't contain any result but just a message saying that product successfully replaced. if replaceBinary && r.StatusCode == 201 { result := new(ResultData) result.Result.ProductID = productID // log.Entry().Debugf("[DEBUG] ===> Return 'replaceBinary && r.StatusCode == 201' from 'UploadScanFile' : %v", result) return result } else { result := new(ResultData) pc.mapResponse(r.Body, result) // log.Entry().Debugf("[DEBUG] ===> Return '!replaceBinary' from 'UploadScanFile' : %v", result) return result } //return result } // DeclareFetchURL configures the fetch url for the protecode scan func (pc *Protecode) DeclareFetchURL(cleanupMode, group, fetchURL, version string, productID int, replaceBinary bool) *ResultData { deleteBinary := (cleanupMode == "binary" || cleanupMode == "complete") var headers = make(map[string][]string) if (replaceBinary) && (version != "") { log.Entry().Debugf("[DEBUG][FETCH_URL] ===> replaceBinary && version != empty ") headers = map[string][]string{"Group": {group}, "Delete-Binary": {fmt.Sprintf("%v", deleteBinary)}, "Replace": {fmt.Sprintf("%v", productID)}, "Version": {version}, "Url": {fetchURL}, "Content-Type": {"application/json"}} } else if replaceBinary { log.Entry().Debugf("[DEBUG][FETCH_URL] ===> replaceBinary") headers = map[string][]string{"Group": {group}, "Delete-Binary": {fmt.Sprintf("%v", deleteBinary)}, "Replace": {fmt.Sprintf("%v", productID)}, "Url": {fetchURL}, "Content-Type": {"application/json"}} } else if version != "" { log.Entry().Debugf("[DEBUG][FETCH_URL] ===> version != empty ") headers = map[string][]string{"Group": {group}, "Delete-Binary": {fmt.Sprintf("%v", deleteBinary)}, "Version": {version}, "Url": {fetchURL}, "Content-Type": {"application/json"}} } else { log.Entry().Debugf("[DEBUG][FETCH_URL] ===> replaceBinary is false and version == empty") headers = map[string][]string{"Group": {group}, "Delete-Binary": {fmt.Sprintf("%v", deleteBinary)}, "Url": {fetchURL}, "Content-Type": {"application/json"}} } // log.Entry().Debugf("[DEBUG] ===> Headers for fetch upload: %v", headers) //headers := map[string][]string{"Group": {group}, "Delete-Binary": {fmt.Sprintf("%v", deleteBinary)}, "Url": {fetchURL}, "Content-Type": {"application/json"}} protecodeURL := fmt.Sprintf("%v/api/fetch/", pc.serverURL) r, statusCode, err := pc.sendAPIRequest(http.MethodPost, protecodeURL, headers) if err != nil { //TODO: bubble up error pc.logger.WithError(err).Fatalf("Error during declare fetch url: %v", protecodeURL) } // log.Entry().Debugf("[DEBUG] ===> Fetch request r: %v", r) // log.Entry().Debugf("[DEBUG] ===> Fetch request r.StatusCode: %v", statusCode) // For replaceBinary option response doesn't contain any result but just a message saying that product successfully replaced. if replaceBinary && statusCode == 201 { result := new(ResultData) result.Result.ProductID = productID // log.Entry().Debugf("[DEBUG] ===> Fetch Return 'replaceBinary && statusCode == 201' from 'DeclareFetchURL' : %v", result) return result } else { result := new(ResultData) pc.mapResponse(*r, result) // log.Entry().Debugf("[DEBUG] ===> Fetch Return '!replaceBinary' from 'DeclareFetchURL' : %v", result) return result } // return result } // 2021-04-20 d : // Found, via web search, an announcement that the set of status codes is expanding from // B, R, F // to // B, R, F, S, D, P. // Only R and F indicate work has completed. func scanInProgress(status string) bool { return status != statusReady && status != statusFailed } //PollForResult polls the protecode scan for the result scan func (pc *Protecode) PollForResult(productID int, timeOutInMinutes string) ResultData { var response ResultData var err error ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() var ticks int64 = 6 if len(timeOutInMinutes) > 0 { parsedTimeOutInMinutes, _ := strconv.ParseInt(timeOutInMinutes, 10, 64) ticks = parsedTimeOutInMinutes * 6 } pc.logger.Infof("Poll for result %v times", ticks) for i := ticks; i > 0; i-- { response, err = pc.pullResult(productID) if err != nil { ticker.Stop() i = 0 return response } if !scanInProgress(response.Result.Status) { ticker.Stop() i = 0 break } select { case t := <-ticker.C: pc.logger.Debugf("Tick : %v Processing status for productID %v", t, productID) } } if scanInProgress(response.Result.Status) { response, err = pc.pullResult(productID) if len(response.Result.Components) < 1 { // 2020-04-20 d : // We are required to scan all images including 3rd party ones. // We have found that Crossplane makes use docker images that contain no // executable code. // So we can no longer treat an empty Components list as an error. pc.logger.Warn("Protecode scan did not identify any components.") } if err != nil || response.Result.Status == statusBusy { //TODO: bubble up error pc.logger.Fatalf("No result after polling err: %v protecode status: %v", err, response.Result.Status) } } return response } func (pc *Protecode) pullResult(productID int) (ResultData, error) { protecodeURL := pc.createURL("/api/product/", fmt.Sprintf("%v/", productID), "") headers := map[string][]string{ "acceptType": {"application/json"}, } r, _, err := pc.sendAPIRequest(http.MethodGet, protecodeURL, headers) if err != nil { return *new(ResultData), err } result := new(ResultData) pc.mapResponse(*r, result) return *result, nil } // verify provided product id func (pc *Protecode) VerifyProductID(ProductID int) bool { // pc.logger.Debugf("[DEBUG] ===> Verification of product id started ..... : %v", ProductID) pc.logger.Infof("Verification of product id (%v) started ... ", ProductID) // TODO: Optimise product id verification _, err := pc.pullResult(ProductID) // If response has an error then we assume this product id doesn't exist or user has no access if err != nil { return false } // Otherwise product exists return true } // LoadExistingProduct loads the existing product from protecode service func (pc *Protecode) LoadExistingProduct(group string, fileName string) int { var productID int = -1 protecodeURL := pc.createURL("/api/apps/", fmt.Sprintf("%v/", group), fileName) headers := map[string][]string{ "acceptType": {"application/json"}, } pc.logger.Debugf("[DEBUG] ===> LoadExistingProduct searching a product (%v) with URL: %v", fileName, protecodeURL) // pc.logger.Infof("[DEBUG] ===> LoadExistingProduct searching a product (%v) with URL: %v", fileName, protecodeURL) response := pc.loadExisting(protecodeURL, headers) // pc.logger.Debugf("[DEBUG] ===> LoadExistingProduct response obj: %v", response) if len(response.Products) > 0 { // pc.logger.Debugf("[DEBUG] ===> LoadExistingProduct: response.Product obj: %v", response.Products) // Highest product id means the latest scan for this particular product, therefore we take a product id with the highest number for i := 0; i < len(response.Products); i++ { // Check filename, it should be the same as we searched if response.Products[i].FileName == fileName { if productID < response.Products[i].ProductID { productID = response.Products[i].ProductID } } } } //productID = response.Products[0].ProductID pc.logger.Debugf("[DEBUG] ===> Re-use existing Protecode scan - group: %v, productID: %v", group, productID) // pc.logger.Infof("Automatic product id detection completed: %v", productID) return productID } // func (pc *Protecode) loadExisting(protecodeURL string, headers map[string][]string) *ProductData { r, _, err := pc.sendAPIRequest(http.MethodGet, protecodeURL, headers) if err != nil { //TODO: bubble up error pc.logger.WithError(err).Fatalf("Error during load existing product: %v", protecodeURL) } result := new(ProductData) pc.mapResponse(*r, result) return result }