package checkmarx import ( "bytes" "encoding/json" "fmt" "io" "io/ioutil" "net/http" "net/url" "strconv" "strings" "time" "encoding/xml" piperHttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) // ReportsDirectory defines the subfolder for the Checkmarx reports which are generated const ReportsDirectory = "checkmarx" // AuthToken - Structure to store OAuth2 token type AuthToken struct { TokenType string `json:"token_type"` AccessToken string `json:"access_token"` ExpiresIn int `json:"expires_in"` } // Preset - Project's Preset type Preset struct { ID int `json:"id"` Name string `json:"name"` OwnerName string `json:"ownerName"` Link Link `json:"link"` } // Scan - Scan Structure type Scan struct { ID int `json:"id"` Link Link `json:"link"` } // ProjectCreateResult - ProjectCreateResult Structure type ProjectCreateResult struct { ID int `json:"id"` Link Link `json:"link"` } // Report - Report Structure type Report struct { ReportID int `json:"reportId"` Links Links `json:"links"` } // ResultsStatistics - ResultsStatistics Structure type ResultsStatistics struct { High int `json:"highSeverity"` Medium int `json:"mediumSeverity"` Low int `json:"lowSeverity"` Info int `json:"infoSeverity"` } // ScanStatus - ScanStatus Structure type ScanStatus struct { ID int `json:"id"` Link Link `json:"link"` Status Status `json:"status"` ScanType string `json:"scanType"` Comment string `json:"comment"` IsIncremental bool `json:"isIncremental"` } // Status - Status Structure type Status struct { ID int `json:"id"` Name string `json:"name"` Details ScanStatusDetail `json:"details"` } // ScanStatusDetail - ScanStatusDetail Structure type ScanStatusDetail struct { Stage string `json:"stage"` Step string `json:"step"` } // ReportStatusResponse - ReportStatusResponse Structure type ReportStatusResponse struct { Location string `json:"location"` ContentType string `json:"contentType"` Status ReportStatus `json:"status"` } // ReportStatus - ReportStatus Structure type ReportStatus struct { ID int `json:"id"` Value string `json:"value"` } // Project - Project Structure type Project struct { ID int `json:"id"` TeamID string `json:"teamId"` Name string `json:"name"` IsPublic bool `json:"isPublic"` SourceSettingsLink SourceSettingsLink `json:"sourceSettingsLink"` Link Link `json:"link"` } // ScanSettings - scan settings at project level type ScanSettings struct { Project ProjectLink `json:"project"` Preset PresetLink `json:"preset"` EngineConfiguration EngineConfigurationLink `json:"engineConfiguration" ` } // ProjectLink - project link found in ScanSettings response type ProjectLink struct { ProjectID int `json:"id"` Link Link `json:"link"` } // PresetLink - preset link found in ScanSettings response type PresetLink struct { PresetID int `json:"id"` Link Link `json:"link"` } // EngineConfigurationLink - engine configuration link found in ScanSettings response type EngineConfigurationLink struct { EngineConfigurationID int `json:"id"` Link Link `json:"link"` } // Team - Team Structure type Team struct { ID json.RawMessage `json:"id"` FullName string `json:"fullName"` } // Links - Links Structure type Links struct { Report Link `json:"report"` Status Link `json:"status"` } // Link - Link Structure type Link struct { Rel string `json:"rel"` URI string `json:"uri"` } // SourceSettingsLink - SourceSettingsLink Structure type SourceSettingsLink struct { Type string `json:"type"` Rel string `json:"rel"` URI string `json:"uri"` } type ShortDescription struct { Text string `json:"shortDescription"` } // DetailedResult - DetailedResult Structure type DetailedResult struct { XMLName xml.Name `xml:"CxXMLResults"` InitiatorName string `xml:"InitiatorName,attr"` ScanID string `xml:"ScanId,attr"` Owner string `xml:"Owner,attr"` ProjectID string `xml:"ProjectId,attr"` ProjectName string `xml:"ProjectName,attr"` TeamFullPathOnReportDate string `xml:"TeamFullPathOnReportDate,attr"` DeepLink string `xml:"DeepLink,attr"` ScanStart string `xml:"ScanStart,attr"` Preset string `xml:"Preset,attr"` ScanTime string `xml:"ScanTime,attr"` LinesOfCodeScanned int `xml:"LinesOfCodeScanned,attr"` FilesScanned int `xml:"FilesScanned,attr"` ReportCreationTime string `xml:"ReportCreationTime,attr"` Team string `xml:"Team,attr"` CheckmarxVersion string `xml:"CheckmarxVersion,attr"` ScanType string `xml:"ScanType,attr"` SourceOrigin string `xml:"SourceOrigin,attr"` Visibility string `xml:"Visibility,attr"` Queries []Query `xml:"Query"` } // Query - Query Structure type Query struct { XMLName xml.Name `xml:"Query"` Name string `xml:"name,attr"` Results []Result `xml:"Result"` } // Result - Result Structure type Result struct { XMLName xml.Name `xml:"Result"` State string `xml:"state,attr"` Severity string `xml:"Severity,attr"` FalsePositive string `xml:"FalsePositive,attr"` } // SystemInstance is the client communicating with the Checkmarx backend type SystemInstance struct { serverURL string username string password string client piperHttp.Uploader logger *logrus.Entry } // System is the interface abstraction of a specific SystemIns type System interface { FilterPresetByName(presets []Preset, presetName string) Preset FilterPresetByID(presets []Preset, presetID int) Preset FilterProjectByName(projects []Project, projectName string) Project FilterTeamByName(teams []Team, teamName string) (Team, error) FilterTeamByID(teams []Team, teamID json.RawMessage) Team DownloadReport(reportID int) ([]byte, error) GetReportStatus(reportID int) (ReportStatusResponse, error) RequestNewReport(scanID int, reportType string) (Report, error) GetResults(scanID int) ResultsStatistics GetScanStatusAndDetail(scanID int) (string, ScanStatusDetail) GetScans(projectID int) ([]ScanStatus, error) ScanProject(projectID int, isIncremental, isPublic, forceScan bool) (Scan, error) UpdateProjectConfiguration(projectID int, presetID int, engineConfigurationID string) error UpdateProjectExcludeSettings(projectID int, excludeFolders string, excludeFiles string) error UploadProjectSourceCode(projectID int, zipFile string) error CreateProject(projectName, teamID string) (ProjectCreateResult, error) CreateBranch(projectID int, branchName string) int GetPresets() []Preset GetProjectByID(projectID int) (Project, error) GetProjectsByNameAndTeam(projectName, teamID string) ([]Project, error) GetProjects() ([]Project, error) GetShortDescription(scanID int, pathID int) (ShortDescription, error) GetTeams() []Team } // NewSystemInstance returns a new Checkmarx client for communicating with the backend func NewSystemInstance(client piperHttp.Uploader, serverURL, username, password string) (*SystemInstance, error) { loggerInstance := log.Entry().WithField("package", "SAP/jenkins-library/pkg/checkmarx") sys := &SystemInstance{ serverURL: serverURL, username: username, password: password, client: client, logger: loggerInstance, } token, err := sys.getOAuth2Token() if err != nil { return sys, errors.Wrap(err, "Error fetching oAuth token") } log.RegisterSecret(token) options := piperHttp.ClientOptions{ Token: token, TransportTimeout: time.Minute * 15, } sys.client.SetOptions(options) return sys, nil } func sendRequest(sys *SystemInstance, method, url string, body io.Reader, header http.Header) ([]byte, error) { return sendRequestInternal(sys, method, url, body, header, []int{}) } func sendRequestInternal(sys *SystemInstance, method, url string, body io.Reader, header http.Header, acceptedErrorCodes []int) ([]byte, error) { var requestBody io.Reader var requestBodyCopy io.Reader if body != nil { closer := ioutil.NopCloser(body) bodyBytes, _ := ioutil.ReadAll(closer) requestBody = bytes.NewBuffer(bodyBytes) requestBodyCopy = bytes.NewBuffer(bodyBytes) defer closer.Close() } response, err := sys.client.SendRequest(method, fmt.Sprintf("%v/cxrestapi%v", sys.serverURL, url), requestBody, header, nil) if err != nil && (response == nil || !piperutils.ContainsInt(acceptedErrorCodes, response.StatusCode)) { sys.recordRequestDetailsInErrorCase(requestBodyCopy, response) sys.logger.Errorf("HTTP request failed with error: %s", err) return nil, err } data, _ := ioutil.ReadAll(response.Body) sys.logger.Debugf("Valid response body: %v", string(data)) defer response.Body.Close() return data, nil } func (sys *SystemInstance) recordRequestDetailsInErrorCase(requestBody io.Reader, response *http.Response) { if requestBody != nil { data, _ := ioutil.ReadAll(ioutil.NopCloser(requestBody)) sys.logger.Errorf("Request body: %s", data) } if response != nil && response.Body != nil { data, _ := ioutil.ReadAll(response.Body) sys.logger.Errorf("Response body: %s", data) response.Body.Close() } } func (sys *SystemInstance) getOAuth2Token() (string, error) { body := url.Values{ "username": {sys.username}, "password": {sys.password}, "grant_type": {"password"}, "scope": {"sast_rest_api"}, "client_id": {"resource_owner_client"}, "client_secret": {"014DF517-39D1-4453-B7B3-9930C563627C"}, } header := http.Header{} header.Add("Content-type", "application/x-www-form-urlencoded") data, err := sendRequest(sys, http.MethodPost, "/auth/identity/connect/token", strings.NewReader(body.Encode()), header) if err != nil { return "", err } var token AuthToken json.Unmarshal(data, &token) return token.TokenType + " " + token.AccessToken, nil } // GetTeams returns the teams the user is assigned to func (sys *SystemInstance) GetTeams() []Team { sys.logger.Debug("Getting Teams...") var teams []Team data, err := sendRequest(sys, http.MethodGet, "/auth/teams", nil, nil) if err != nil { sys.logger.Errorf("Fetching teams failed: %s", err) return teams } json.Unmarshal(data, &teams) return teams } // GetProjects returns the projects defined in the Checkmarx backend which the user has access to func (sys *SystemInstance) GetProjects() ([]Project, error) { return sys.GetProjectsByNameAndTeam("", "") } // GetProjectByID returns the project addressed by projectID from the Checkmarx backend which the user has access to func (sys *SystemInstance) GetProjectByID(projectID int) (Project, error) { sys.logger.Debugf("Getting Project with ID %v...", projectID) var project Project data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/projects/%v", projectID), nil, nil) if err != nil { return project, errors.Wrapf(err, "fetching project %v failed", projectID) } json.Unmarshal(data, &project) return project, nil } // GetProjectsByNameAndTeam returns the project addressed by projectID from the Checkmarx backend which the user has access to func (sys *SystemInstance) GetProjectsByNameAndTeam(projectName, teamID string) ([]Project, error) { sys.logger.Debugf("Getting projects with name %v of team %v...", projectName, teamID) var projects []Project header := http.Header{} header.Set("Accept-Type", "application/json") var data []byte var err error if len(teamID) > 0 && len(projectName) > 0 { body := url.Values{ "projectName": {projectName}, "teamId": {teamID}, } data, err = sendRequestInternal(sys, http.MethodGet, fmt.Sprintf("/projects?%v", body.Encode()), nil, header, []int{404}) } else { data, err = sendRequestInternal(sys, http.MethodGet, "/projects", nil, header, []int{404}) } if err != nil { return projects, errors.Wrapf(err, "fetching project %v failed", projectName) } json.Unmarshal(data, &projects) return projects, nil } // CreateProject creates a new project in the Checkmarx backend func (sys *SystemInstance) CreateProject(projectName, teamID string) (ProjectCreateResult, error) { var result ProjectCreateResult jsonData := map[string]interface{}{ "name": projectName, "owningTeam": teamID, "isPublic": true, } jsonValue, err := json.Marshal(jsonData) if err != nil { return result, errors.Wrapf(err, "failed to marshal project data") } header := http.Header{} header.Set("Content-Type", "application/json") data, err := sendRequest(sys, http.MethodPost, "/projects", bytes.NewBuffer(jsonValue), header) if err != nil { return result, errors.Wrapf(err, "failed to create project %v", projectName) } json.Unmarshal(data, &result) return result, nil } // CreateBranch creates a branch of an existing project in the Checkmarx backend func (sys *SystemInstance) CreateBranch(projectID int, branchName string) int { jsonData := map[string]interface{}{ "name": branchName, } jsonValue, err := json.Marshal(jsonData) if err != nil { sys.logger.Errorf("Error Marshal: %s", err) return 0 } header := http.Header{} header.Set("Content-Type", "application/json") data, err := sendRequest(sys, http.MethodPost, fmt.Sprintf("/projects/%v/branch", projectID), bytes.NewBuffer(jsonValue), header) if err != nil { sys.logger.Errorf("Failed to create project: %s", err) return 0 } var scan Scan json.Unmarshal(data, &scan) return scan.ID } // UploadProjectSourceCode zips and uploads the project sources for scanning func (sys *SystemInstance) UploadProjectSourceCode(projectID int, zipFile string) error { sys.logger.Debug("Starting to upload files...") header := http.Header{} header.Add("Accept-Encoding", "gzip,deflate") header.Add("Accept", "text/plain") resp, err := sys.client.UploadFile(fmt.Sprintf("%v/cxrestapi/projects/%v/sourceCode/attachments", sys.serverURL, projectID), zipFile, "zippedSource", header, nil, "form") if err != nil { return errors.Wrap(err, "failed to uploaded zipped sources") } data, err := ioutil.ReadAll(resp.Body) defer resp.Body.Close() if err != nil { return errors.Wrap(err, "error reading the response data") } responseData := make(map[string]string) json.Unmarshal(data, &responseData) if resp.StatusCode == http.StatusNoContent { return nil } sys.logger.Debugf("Body %s", data) return errors.Wrapf(err, "error writing the request's body, status: %s", resp.Status) } // UpdateProjectExcludeSettings updates the exclude configuration of the project func (sys *SystemInstance) UpdateProjectExcludeSettings(projectID int, excludeFolders string, excludeFiles string) error { jsonData := map[string]string{ "excludeFoldersPattern": excludeFolders, "excludeFilesPattern": excludeFiles, } jsonValue, err := json.Marshal(jsonData) if err != nil { return errors.Wrap(err, "error marhalling project exclude settings") } header := http.Header{} header.Set("Content-Type", "application/json") _, err = sendRequest(sys, http.MethodPut, fmt.Sprintf("/projects/%v/sourceCode/excludeSettings", projectID), bytes.NewBuffer(jsonValue), header) if err != nil { return errors.Wrap(err, "request to checkmarx system failed") } return nil } // GetPresets loads the preset values defined in the Checkmarx backend func (sys *SystemInstance) GetPresets() []Preset { sys.logger.Debug("Getting Presets...") var presets []Preset data, err := sendRequest(sys, http.MethodGet, "/sast/presets", nil, nil) if err != nil { sys.logger.Errorf("Fetching presets failed: %s", err) return presets } json.Unmarshal(data, &presets) return presets } // UpdateProjectConfiguration updates the configuration of the project addressed by projectID func (sys *SystemInstance) UpdateProjectConfiguration(projectID int, presetID int, engineConfigurationID string) error { engineConfigID, _ := strconv.Atoi(engineConfigurationID) var projectScanSettings ScanSettings header := http.Header{} header.Set("Content-Type", "application/json") data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/sast/scanSettings/%v", projectID), nil, header) if err != nil { // if an error happens, try to update the config anyway sys.logger.Warnf("Failed to fetch scan settings of project %v: %s", projectID, err) } else { // Check if the current project config needs to be updated json.Unmarshal(data, &projectScanSettings) if projectScanSettings.Preset.PresetID == presetID && (projectScanSettings.EngineConfiguration.EngineConfigurationID == engineConfigID || engineConfigID == 0) { sys.logger.Debugf("Project configuration does not need to be updated") return nil } } // use the project-level value to configure the project if no value was provided in piper config if engineConfigID == 0 { engineConfigID = projectScanSettings.EngineConfiguration.EngineConfigurationID } jsonData := map[string]interface{}{ "projectId": projectID, "presetId": presetID, "engineConfigurationId": engineConfigID, } jsonValue, err := json.Marshal(jsonData) if err != nil { return errors.Wrapf(err, "error marshalling project data") } _, err = sendRequest(sys, http.MethodPost, "/sast/scanSettings", bytes.NewBuffer(jsonValue), header) if err != nil { return errors.Wrapf(err, "request to checkmarx system failed") } sys.logger.Debugf("Project configuration updated") return nil } // ScanProject triggers a scan on the project addressed by projectID func (sys *SystemInstance) ScanProject(projectID int, isIncremental, isPublic, forceScan bool) (Scan, error) { scan := Scan{} jsonData := map[string]interface{}{ "projectId": projectID, "isIncremental": isIncremental, "isPublic": isPublic, "forceScan": forceScan, "comment": "Scan From Golang Script", } jsonValue, _ := json.Marshal(jsonData) header := http.Header{} header.Set("cxOrigin", "GolangScript") header.Set("Content-Type", "application/json") data, err := sendRequest(sys, http.MethodPost, "/sast/scans", bytes.NewBuffer(jsonValue), header) if err != nil { sys.logger.Errorf("Failed to trigger scan of project %v: %s", projectID, err) return scan, errors.Wrapf(err, "Failed to trigger scan of project %v", projectID) } json.Unmarshal(data, &scan) return scan, nil } // GetScans returns all scan status on the project addressed by projectID func (sys *SystemInstance) GetScans(projectID int) ([]ScanStatus, error) { scans := []ScanStatus{} body := url.Values{ "projectId": {fmt.Sprintf("%v", projectID)}, "last": {fmt.Sprintf("%v", 20)}, } header := http.Header{} header.Set("cxOrigin", "GolangScript") header.Set("Accept-Type", "application/json") data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/sast/scans?%v", body.Encode()), nil, header) if err != nil { sys.logger.Errorf("Failed to fetch scans of project %v: %s", projectID, err) return scans, errors.Wrapf(err, "failed to fetch scans of project %v", projectID) } json.Unmarshal(data, &scans) return scans, nil } // GetScanStatusAndDetail returns the status of the scan addressed by scanID func (sys *SystemInstance) GetScanStatusAndDetail(scanID int) (string, ScanStatusDetail) { var scanStatus ScanStatus data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/sast/scans/%v", scanID), nil, nil) if err != nil { sys.logger.Errorf("Failed to get scan status for scanID %v: %s", scanID, err) return "Failed", ScanStatusDetail{} } json.Unmarshal(data, &scanStatus) return scanStatus.Status.Name, scanStatus.Status.Details } // GetResults returns the results of the scan addressed by scanID func (sys *SystemInstance) GetResults(scanID int) ResultsStatistics { var results ResultsStatistics data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/sast/scans/%v/resultsStatistics", scanID), nil, nil) if err != nil { sys.logger.Errorf("Failed to fetch scan results for scanID %v: %s", scanID, err) return results } json.Unmarshal(data, &results) return results } // RequestNewReport triggers the generation of a report for a specific scan addressed by scanID func (sys *SystemInstance) RequestNewReport(scanID int, reportType string) (Report, error) { report := Report{} jsonData := map[string]interface{}{ "scanId": scanID, "reportType": reportType, "comment": "Scan report triggered by Piper", } jsonValue, _ := json.Marshal(jsonData) header := http.Header{} header.Set("cxOrigin", "GolangScript") header.Set("Content-Type", "application/json") data, err := sendRequest(sys, http.MethodPost, "/reports/sastScan", bytes.NewBuffer(jsonValue), header) if err != nil { return report, errors.Wrapf(err, "Failed to trigger report generation for scan %v", scanID) } json.Unmarshal(data, &report) return report, nil } // GetReportStatus returns the status of the report generation process func (sys *SystemInstance) GetReportStatus(reportID int) (ReportStatusResponse, error) { var response ReportStatusResponse header := http.Header{} header.Set("Accept", "application/json") data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/reports/sastScan/%v/status", reportID), nil, header) if err != nil { sys.logger.Errorf("Failed to fetch report status for reportID %v: %s", reportID, err) return response, errors.Wrapf(err, "failed to fetch report status for reportID %v", reportID) } json.Unmarshal(data, &response) return response, nil } // GetShortDescription returns the short description for an issue with a scanID and pathID func (sys *SystemInstance) GetShortDescription(scanID int, pathID int) (ShortDescription, error) { var shortDescription ShortDescription data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/sast/scans/%v/results/%v/shortDescription", scanID, pathID), nil, nil) if err != nil { sys.logger.Errorf("Failed to get short description for scanID %v and pathID %v: %s", scanID, pathID, err) return shortDescription, err } json.Unmarshal(data, &shortDescription) return shortDescription, nil } // DownloadReport downloads the report addressed by reportID and returns the XML contents func (sys *SystemInstance) DownloadReport(reportID int) ([]byte, error) { header := http.Header{} header.Set("Accept", "application/json") data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/reports/sastScan/%v", reportID), nil, header) if err != nil { return []byte{}, errors.Wrapf(err, "failed to download report with reportID %v", reportID) } return data, nil } // FilterTeamByName filters a team by its name func (sys *SystemInstance) FilterTeamByName(teams []Team, teamName string) (Team, error) { for _, team := range teams { if team.FullName == teamName || team.FullName == strings.ReplaceAll(teamName, `\`, `/`) { return team, nil } } return Team{}, errors.New("Failed to find team with name " + teamName) } // FilterTeamByID filters a team by its ID func (sys *SystemInstance) FilterTeamByID(teams []Team, teamID json.RawMessage) Team { teamIDBytes1, _ := teamID.MarshalJSON() for _, team := range teams { teamIDBytes2, _ := team.ID.MarshalJSON() if bytes.Compare(teamIDBytes1, teamIDBytes2) == 0 { return team } } return Team{} } // FilterProjectByName filters a project by its name func (sys *SystemInstance) FilterProjectByName(projects []Project, projectName string) Project { for _, project := range projects { if project.Name == projectName { sys.logger.Debugf("Filtered project with name %v", project.Name) return project } } return Project{} } // FilterPresetByName filters a preset by its name func (sys *SystemInstance) FilterPresetByName(presets []Preset, presetName string) Preset { for _, preset := range presets { if preset.Name == presetName { return preset } } return Preset{} } // FilterPresetByID filters a preset by its name func (sys *SystemInstance) FilterPresetByID(presets []Preset, presetID int) Preset { for _, preset := range presets { if preset.ID == presetID { return preset } } return Preset{} }