1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2024-12-12 10:55:20 +02:00
sap-jenkins-library/pkg/checkmarxone/checkmarxone.go
michaelkubiaczyk bc8d5efe46
Cxone release supporting applications (#4548)
* Initial in progress

* compiling but not yet functional

* Missed file

* updated checkmarxone step

* Working up to fetching a project then breaks

* Missed file

* Breaks when retrieving projects+proxy set

* Create project & run scan working, now polling

* Fixed polling

* added back the zipfile remove command

* Fixed polling again

* Generates and downloads PDF report

* Updated and working, prep for refactor

* Added compliance steps

* Cleanup, reporting, added groovy connector

* fixed groovy file

* checkmarxone to checkmarxOne

* checkmarxone to checkmarxOne

* split credentials (id+secret, apikey), renamed pullrequestname to branch, groovy fix

* Fixed filenames & yaml

* missed the metadata_generated.go

* added json to sarif conversion

* fix:type in new checkmarxone package

* fix:type in new checkmarxone package

* removed test logs, added temp error log for creds

* extra debugging to fix crash

* improved auth logging, fixed query parse issue

* fixed bug with group fetch when using oauth user

* CWE can be -1 if not defined, can't be uint

* Query also had CweID

* Disabled predicates-fetch in sarif generation

* Removing leftover info log message

* Better error handling

* fixed default preset configuration

* removing .bat files - sorry

* Cleanup per initial review

* refactoring per Gist, fixed project find, add apps

* small fix - sorry for commit noise while testing

* Fixing issues with incremental scans.

* removing maxretries

* Updated per PR feedback, further changes todo toda

* JSON Report changes and reporting cleanup

* removing .bat (again?)

* adding docs, groovy unit test, linter fixes

* Started adding tests maybe 15% covered

* fix(checkmarxOne): test cases for pkg and reporting

* fix(checkmarxOne):fix formatting

* feat(checkmarxone): update interface with missing method

* feat(checkmarxone):change runStep signature to be able to inject dependency

* feat(checkmarxone): add tests for step (wip)

* Adding a bit more coverage

* feat(checkmarxOne): fix code review

* feat(checkmarxOne): fix code review

* feat(checkmarxOne): fix code review

* feat(checkmarxOne): fix integration test PR

* adding scan-summary bug workaround, reportgen fail

* enforceThresholds fix when no results passed in

* fixed gap when preset empty in yaml & project conf

* fixed another gap in preset selection

* fix 0-result panic

* fail when no preset is set anywhere

* removed comment

* initial project-under-app support

* fixing sarif reportgen

* some cleanup of error messages

* post-merge test fixes

* revert previous upstream merge

* fix:formatting

* fix(checkmarxOne):yamllint too many blank lines

* fix(checkmarxOne):unit test

* fix(checkmarxOne):generated code

---------

Co-authored-by: thtri <trinhthanhhai@gmail.com>
Co-authored-by: Thanh-Hai Trinh <thanh.hai.trinh@sap.com>
2023-09-05 21:49:27 +02:00

1362 lines
42 KiB
Go

package checkmarxOne
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
//"strconv"
"strings"
"time"
//"encoding/xml"
piperHttp "github.com/SAP/jenkins-library/pkg/http"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
// ReportsDirectory defines the subfolder for the Checkmarx reports which are generated
const ReportsDirectory = "checkmarxOne"
const cxOrigin = "GolangScript"
// AuthToken - Structure to store OAuth2 token
// Updated for Cx1
type AuthToken struct {
TokenType string `json:"token_type"`
AccessToken string `json:"access_token"`
ExpiresIn int `json:"expires_in"`
// RefreshExpiresIn int `json:"refresh_expires_in"`
// NotBeforePolicy int `json:"not-before-policy"`
// Scope string `json:"scope"`
}
type Application struct {
ApplicationID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Criticality uint `json:"criticality"`
Rules []ApplicationRule `json:"rules"`
Tags map[string]string `json:"tags"`
CreatedAt string `json:"createdAt"`
UpdatedAt string `json:"updatedAt"`
}
type ApplicationRule struct {
Type string `json:"type"`
Value string `json:"value"`
}
// Preset - Project's Preset
// Updated for Cx1
type Preset struct {
PresetID int `json:"id"`
Name string `json:"name"`
}
// Project - Project Structure
// Updated for Cx1
type Project struct {
ProjectID string `json:"id"`
Name string `json:"name"`
CreatedAt string `json:"createdAt"`
UpdatedAt string `json:"updatedAt"`
Groups []string `json:"groups"`
Applications []string `json:"applicationIds"`
Tags map[string]string `json:"tags"`
RepoUrl string `json:"repoUrl"`
MainBranch string `json:"mainBranch"`
Origin string `json:"origin"`
Criticality int `json:"criticality"`
}
// New for Cx1
// These settings are higher-level settings that define how an engine should run, for example "multi-language" mode or setting a preset.
type ProjectConfigurationSetting struct {
Key string `json:"key"`
Name string `json:"name"`
Category string `json:"category"`
OriginLevel string `json:"originLevel"`
Value string `json:"value"`
ValueType string `json:"valuetype"`
ValueTypeParams string `json:"valuetypeparams"`
AllowOverride bool `json:"allowOverride"`
}
type Query struct {
QueryID uint64 `json:"queryID,string"`
Name string `json:"queryName"`
Group string
Language string
Severity string
CweID int64
QueryDescriptionID int64
Custom bool
}
// ReportStatus - ReportStatus Structure
// Updated for Cx1
type ReportStatus struct {
ReportID string `json:"reportId"`
Status string `json:"status"`
ReportURL string `json:"url"`
}
type ResultsPredicates struct {
PredicateID string `json:"ID"`
SimilarityID int64 `json:"similarityId,string"`
ProjectID string `json:"projectId"`
State string `json:"state"`
Comment string `json:"comment"`
Severity string `json:"severity"`
CreatedBy string
CreatedAt string
}
// Scan - Scan Structure
// updated for Cx1
type Scan struct {
ScanID string `json:"id"`
Status string `json:"status"`
StatusDetails []ScanStatusDetails `json:"statusDetails"`
Branch string `json:"branch"`
CreatedAt string `json:"createdAt"`
UpdatedAt string `json:"updatedAt"`
ProjectID string `json:"projectId"`
ProjectName string `json:"projectName"`
UserAgent string `json:"userAgent"`
Initiator string `json:"initiator"`
Tags map[string]string `json:"tags"`
Metadata struct {
Type string `json:"type"`
Configs []ScanConfiguration `json:"configs"`
} `json:"metadata"`
Engines []string `json:"engines"`
SourceType string `json:"sourceType"`
SourceOrigin string `json:"sourceOrigin"`
}
// New for Cx1: ScanConfiguration - list of key:value pairs used to configure the scan for each scan engine
// This is specifically for scan-level configurations like "is incremental" and scan tags
type ScanConfiguration struct {
ScanType string `json:"type"`
Values map[string]string `json:"value"`
}
/*
{"scanId":"bef5d38b-7eb9-4138-b74b-2639fcf49e2e","projectId":"ad34ade3-9bf3-4b5a-91d7-3ad67eca7852","loc":137,"fileCount":12,"isIncremental":false,"isIncrementalCanceled":false,"queryPreset":"ASA Premium"}
*/
type ScanMetadata struct {
ScanID string
ProjectID string
LOC int
FileCount int
IsIncremental bool
IsIncrementalCanceled bool
PresetName string `json:"queryPreset"`
}
type ScanResultData struct {
QueryID uint64
QueryName string
Group string
ResultHash string
LanguageName string
Nodes []ScanResultNodes
}
type ScanResultNodes struct {
ID string
Line int
Name string
Column int
Length int
Method string
NodeID int
DOMType string
FileName string
FullName string
TypeName string
MethodLine int
Definitions string
}
type ScanResult struct {
Type string
ResultID string `json:"id"`
SimilarityID int64 `json:"similarityId,string"`
Status string
State string
Severity string
CreatedAt string `json:"created"`
FirstFoundAt string
FoundAt string
FirstScanId string
Description string
Data ScanResultData
VulnerabilityDetails ScanResultDetails
}
type ScanResultDetails struct {
CweId int
Compliances []string
}
// Cx1: StatusDetails - details of each engine type's scan status for a multi-engine scan
type ScanStatusDetails struct {
Name string `json:"name"`
Status string `json:"status"`
Details string `json:"details"`
}
// Very simplified for now
type ScanSummary struct {
TenantID string
ScanID string
SASTCounters struct {
//QueriesCounters []?
//SinkFileCounters []?
LanguageCounters []struct {
Language string
Counter uint64
}
ComplianceCounters []struct {
Compliance string
Counter uint64
}
SeverityCounters []struct {
Severity string
Counter uint64
}
StatusCounters []struct {
Status string
Counter uint64
}
StateCounters []struct {
State string
Counter uint64
}
TotalCounter uint64
FilesScannedCounter uint64
}
// ignoring the other counters
// KICSCounters
// SCACounters
// SCAPackagesCounters
// SCAContainerCounters
// APISecCounters
}
// Status - Status Structure
type Status struct {
ID int `json:"id"`
Name string `json:"name"`
Details ScanStatusDetails `json:"details"`
}
type WorkflowLog struct {
Source string `json:"Source"`
Info string `json:"Info"`
Timestamp string `json:"Timestamp"`
}
// Cx1 Group/Group - Group Structure
type Group struct {
GroupID string `json:"id"`
Name string `json:"name"`
}
// SystemInstance is the client communicating with the Checkmarx backend
type SystemInstance struct {
serverURL string
iamURL string // New for Cx1
tenant string // New for Cx1
APIKey string // New for Cx1
oauth_client_id string // separate from APIKey
oauth_client_secret string //separate from APIKey
client piperHttp.Uploader
logger *logrus.Entry
}
// System is the interface abstraction of a specific SystemIns
type System interface {
DownloadReport(reportID string) ([]byte, error)
GetReportStatus(reportID string) (ReportStatus, error)
RequestNewReport(scanID, projectID, branch, reportType string) (string, error)
CreateApplication(appname string) (Application, error)
GetApplicationByName(appname string) (Application, error)
GetApplicationByID(appId string) (Application, error)
UpdateApplication(app *Application) error
GetScan(scanID string) (Scan, error)
GetScanMetadata(scanID string) (ScanMetadata, error)
GetScanResults(scanID string, limit uint64) ([]ScanResult, error)
GetScanSummary(scanID string) (ScanSummary, error)
GetResultsPredicates(SimilarityID int64, ProjectID string) ([]ResultsPredicates, error)
GetScanWorkflow(scanID string) ([]WorkflowLog, error)
GetLastScans(projectID string, limit int) ([]Scan, error)
GetLastScansByStatus(projectID string, limit int, status []string) ([]Scan, error)
ScanProject(projectID, sourceUrl, branch, scanType string, settings []ScanConfiguration) (Scan, error)
ScanProjectZip(projectID, sourceUrl, branch string, settings []ScanConfiguration) (Scan, error)
ScanProjectGit(projectID, repoUrl, branch string, settings []ScanConfiguration) (Scan, error)
UploadProjectSourceCode(projectID string, zipFile string) (string, error)
CreateProject(projectName string, groupIDs []string) (Project, error)
CreateProjectInApplication(projectName, applicationID string, groupIDs []string) (Project, error)
GetPresets() ([]Preset, error)
GetProjectByID(projectID string) (Project, error)
GetProjectsByName(projectName string) ([]Project, error)
GetProjectsByNameAndGroup(projectName, groupID string) ([]Project, error)
GetProjects() ([]Project, error)
GetQueries() ([]Query, error)
//GetShortDescription(scanID int, pathID int) (ShortDescription, error)
GetGroups() ([]Group, error)
GetGroupByName(groupName string) (Group, error)
GetGroupByID(groupID string) (Group, error)
SetProjectBranch(projectID, branch string, allowOverride bool) error
SetProjectPreset(projectID, presetName string, allowOverride bool) error
SetProjectLanguageMode(projectID, languageMode string, allowOverride bool) error
SetProjectFileFilter(projectID, filter string, allowOverride bool) error
GetProjectConfiguration(projectID string) ([]ProjectConfigurationSetting, error)
UpdateProjectConfiguration(projectID string, settings []ProjectConfigurationSetting) error
}
// NewSystemInstance returns a new Checkmarx client for communicating with the backend
// Updated for Cx1
func NewSystemInstance(client piperHttp.Uploader, serverURL, iamURL, tenant, APIKey, client_id, client_secret string) (*SystemInstance, error) {
loggerInstance := log.Entry().WithField("package", "SAP/jenkins-library/pkg/checkmarxOne")
sys := &SystemInstance{
serverURL: serverURL,
iamURL: iamURL,
tenant: tenant,
APIKey: APIKey,
oauth_client_id: client_id,
oauth_client_secret: client_secret,
client: client,
logger: loggerInstance,
}
var token string
var err error
if APIKey != "" {
token, err = sys.getAPIToken()
if err != nil {
return sys, errors.Wrap(err, fmt.Sprintf("Error fetching oAuth token using API Key: %v", shortenGUID(APIKey)))
}
} else if client_id != "" && client_secret != "" {
token, err = sys.getOAuth2Token()
if err != nil {
return sys, errors.Wrap(err, fmt.Sprintf("Error fetching oAuth token using OIDC client: %v/%v", shortenGUID(client_id), shortenGUID(client_secret)))
}
} else {
return sys, errors.New("No APIKey or client_id/client_secret provided.")
}
log.RegisterSecret(token)
options := piperHttp.ClientOptions{
Token: token,
TransportTimeout: time.Minute * 15,
}
sys.client.SetOptions(options)
return sys, nil
}
// Updated for Cx1
func sendRequest(sys *SystemInstance, method, url string, body io.Reader, header http.Header, acceptedErrorCodes []int) ([]byte, error) {
cx1url := fmt.Sprintf("%v/api%v", sys.serverURL, url)
return sendRequestInternal(sys, method, cx1url, body, header, acceptedErrorCodes)
}
// Updated for Cx1
func sendRequestIAM(sys *SystemInstance, method, base, url string, body io.Reader, header http.Header, acceptedErrorCodes []int) ([]byte, error) {
iamurl := fmt.Sprintf("%v%v/realms/%v%v", sys.iamURL, base, sys.tenant, url)
return sendRequestInternal(sys, method, iamurl, body, header, acceptedErrorCodes)
}
// Updated for Cx1
func sendRequestInternal(sys *SystemInstance, method, url string, body io.Reader, header http.Header, acceptedErrorCodes []int) ([]byte, error) {
var requestBody io.Reader
var reqBody string
if body != nil {
closer := io.NopCloser(body)
bodyBytes, _ := io.ReadAll(closer)
reqBody = string(bodyBytes)
requestBody = bytes.NewBuffer(bodyBytes)
defer closer.Close()
}
if header == nil {
header = http.Header{}
}
header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0")
//header.Set("User-Agent", "Project-Piper.io cicd pipeline") // currently this causes some requests to fail due to unknown UA validation in the backend.
response, err := sys.client.SendRequest(method, url, requestBody, header, nil)
if err != nil && (response == nil || !piperutils.ContainsInt(acceptedErrorCodes, response.StatusCode)) {
var resBodyBytes []byte
if response != nil && response.Body != nil {
resBodyBytes, _ = io.ReadAll(response.Body)
defer response.Body.Close()
resBody := string(resBodyBytes)
sys.recordRequestDetailsInErrorCase(reqBody, resBody)
}
var str string
if len(resBodyBytes) > 0 {
var msg map[string]interface{}
_ = json.Unmarshal(resBodyBytes, &msg)
if msg["message"] != nil {
str = msg["message"].(string)
} else if msg["error_description"] != nil {
str = msg["error_description"].(string)
} else if msg["error"] != nil {
str = msg["error"].(string)
} else {
if len(str) > 20 {
str = string(resBodyBytes[:20])
} else {
str = string(resBodyBytes)
}
}
}
if err != nil {
if str != "" {
err = fmt.Errorf("%s: %v", err, str)
} else {
err = fmt.Errorf("%s", err)
}
sys.logger.Errorf("HTTP request failed with error: %s", err)
return resBodyBytes, err
} else {
if str != "" {
err = fmt.Errorf("HTTP %v: %v", response.Status, str)
} else {
err = fmt.Errorf("HTTP %v", response.Status)
}
sys.logger.Errorf("HTTP response indicates error: %s", err)
return resBodyBytes, err
}
}
data, _ := io.ReadAll(response.Body)
//sys.logger.Debugf("Valid response body: %v", string(data))
defer response.Body.Close()
return data, nil
}
func (sys *SystemInstance) recordRequestDetailsInErrorCase(requestBody string, responseBody string) {
if requestBody != "" {
sys.logger.Errorf("Request body: %s", requestBody)
}
if responseBody != "" {
sys.logger.Errorf("Response body: %s", responseBody)
}
}
/*
CxOne authentication options are:
1. APIKey: post to /protocol/openid-connect/token with client_id ("ast-app"), refresh_token (APIKey generated in the UI), & grant_type ("refresh_token")
2. OAuth Client (service account): post to /protocol/openid-connect/token with client_id (set in the OIDC client in the Cx1 UI), client_secret (set in the UI), grant_type ("client_credentials")
For regular users, the API Key is likely to be used - it is a token tied to the user account.
For service accounts, Administrators will need to create OAuth clients.
*/
// Updated for Cx1
func (sys *SystemInstance) getOAuth2Token() (string, error) {
body := url.Values{
"grant_type": {"client_credentials"},
"client_id": {sys.oauth_client_id},
"client_secret": {sys.oauth_client_secret},
}
header := http.Header{}
header.Add("Content-type", "application/x-www-form-urlencoded")
data, err := sendRequestIAM(sys, http.MethodPost, "/auth", "/protocol/openid-connect/token", strings.NewReader(body.Encode()), header, []int{})
if err != nil {
return "", err
}
var token AuthToken
json.Unmarshal(data, &token)
return token.TokenType + " " + token.AccessToken, nil
}
// Updated for Cx1
func (sys *SystemInstance) getAPIToken() (string, error) {
body := url.Values{
"grant_type": {"refresh_token"},
"client_id": {"ast-app"},
"refresh_token": {sys.APIKey},
}
header := http.Header{}
header.Add("Content-type", "application/x-www-form-urlencoded")
data, err := sendRequestIAM(sys, http.MethodPost, "/auth", "/protocol/openid-connect/token", strings.NewReader(body.Encode()), header, []int{})
if err != nil {
return "", err
}
var token AuthToken
json.Unmarshal(data, &token)
return token.TokenType + " " + token.AccessToken, nil
}
func (sys *SystemInstance) GetApplicationsByName(name string, limit uint64) ([]Application, error) {
sys.logger.Debugf("Get Cx1 Applications by name: %v", name)
var ApplicationResponse struct {
TotalCount uint64
FilteredCount uint64
Applications []Application
}
body := url.Values{
//"offset": {fmt.Sprintf("%d", 0)},
"limit": {fmt.Sprintf("%d", limit)},
"name": {name},
}
response, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/applications?%v", body.Encode()), nil, nil, []int{})
if err != nil {
return ApplicationResponse.Applications, err
}
err = json.Unmarshal(response, &ApplicationResponse)
sys.logger.Tracef("Retrieved %d applications", len(ApplicationResponse.Applications))
return ApplicationResponse.Applications, err
}
func (sys *SystemInstance) GetApplicationByID(appId string) (Application, error) {
sys.logger.Debugf("Get Cx1 Application by ID: %v", appId)
var ret Application
response, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/applications/%v", appId), nil, nil, []int{})
if err != nil {
return ret, err
}
err = json.Unmarshal(response, &ret)
return ret, err
}
func (sys *SystemInstance) GetApplicationByName(name string) (Application, error) {
apps, err := sys.GetApplicationsByName(name, 0)
if err != nil {
return Application{}, err
}
for _, a := range apps {
if a.Name == name {
return a, nil
}
}
return Application{}, fmt.Errorf("no application found named %v", name)
}
func (sys *SystemInstance) CreateApplication(appname string) (Application, error) {
sys.logger.Debugf("Create Application: %v", appname)
data := map[string]interface{}{
"name": appname,
"description": "",
"criticality": 3,
"rules": []ApplicationRule{},
"tags": map[string]string{},
}
var app Application
jsonBody, err := json.Marshal(data)
if err != nil {
return app, err
}
response, err := sendRequest(sys, http.MethodPost, "/applications", bytes.NewReader(jsonBody), nil, []int{})
if err != nil {
sys.logger.Tracef("Error while creating application: %s", err)
return app, err
}
err = json.Unmarshal(response, &app)
return app, err
}
func (a *Application) GetRuleByType(ruletype string) *ApplicationRule {
for id := range a.Rules {
if a.Rules[id].Type == ruletype {
return &(a.Rules[id])
}
}
return nil
}
func (a *Application) AddRule(ruletype, value string) {
rule := a.GetRuleByType(ruletype)
if rule == nil {
var newrule ApplicationRule
newrule.Type = ruletype
newrule.Value = value
a.Rules = append(a.Rules, newrule)
} else {
if rule.Value == value || strings.Contains(rule.Value, fmt.Sprintf(";%v;", value)) || rule.Value[len(rule.Value)-len(value)-1:] == fmt.Sprintf(";%v", value) || rule.Value[:len(value)+1] == fmt.Sprintf("%v;", value) {
return // rule value already contains this value
}
rule.Value = fmt.Sprintf("%v;%v", rule.Value, value)
}
}
func (a *Application) AssignProject(project *Project) {
a.AddRule("project.name.in", project.Name)
}
func (sys *SystemInstance) UpdateApplication(app *Application) error {
sys.logger.Debugf("Updating application: %v", app.Name)
jsonBody, err := json.Marshal(*app)
if err != nil {
return err
}
_, err = sendRequest(sys, http.MethodPut, fmt.Sprintf("/applications/%v", app.ApplicationID), bytes.NewReader(jsonBody), nil, []int{})
if err != nil {
sys.logger.Tracef("Error while updating application: %s", err)
return err
}
return nil
}
// Updated for Cx1
func (sys *SystemInstance) GetGroups() ([]Group, error) {
sys.logger.Debug("Getting Groups...")
var groups []Group
data, err := sendRequestIAM(sys, http.MethodGet, "/auth", "/pip/groups", nil, http.Header{}, []int{})
if err != nil {
sys.logger.Errorf("Fetching groups failed: %s", err)
return groups, err
}
err = json.Unmarshal(data, &groups)
if err != nil {
sys.logger.Errorf("Fetching groups failed: %s", err)
return groups, err
}
return groups, nil
}
// New for Cx1
func (sys *SystemInstance) GetGroupByName(groupName string) (Group, error) {
sys.logger.Debugf("Getting Group named %v...", groupName)
groups, err := sys.GetGroups()
var group Group
if err != nil {
return group, err
}
for _, g := range groups {
if g.Name == groupName {
return g, nil
}
}
return group, errors.New(fmt.Sprintf("No group matching %v", groupName))
}
// New for Cx1
func (sys *SystemInstance) GetGroupByID(groupID string) (Group, error) {
sys.logger.Debugf("Getting Group with ID %v...", groupID)
groups, err := sys.GetGroups()
var group Group
if err != nil {
return group, err
}
for _, g := range groups {
if g.GroupID == groupID {
return g, nil
}
}
return group, errors.New(fmt.Sprintf("No group with ID %v", groupID))
}
// GetProjects returns the projects defined in the Checkmarx backend which the user has access to
func (sys *SystemInstance) GetProjects() ([]Project, error) {
return sys.GetProjectsByNameAndGroup("", "")
}
// GetProjectByID returns the project addressed by projectID from the Checkmarx backend which the user has access to
// Updated for Cx1
func (sys *SystemInstance) GetProjectByID(projectID string) (Project, error) {
sys.logger.Debugf("Getting Project with ID %v...", projectID)
var project Project
data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/projects/%v", projectID), nil, http.Header{}, []int{})
if err != nil {
return project, errors.Wrapf(err, "fetching project %v failed", projectID)
}
err = json.Unmarshal(data, &project)
return project, err
}
// GetProjectsByNameAndGroup returns the project addressed by project name from the Checkmarx backend which the user has access to
// Updated for Cx1
func (sys *SystemInstance) GetProjectsByName(projectName string) ([]Project, error) {
sys.logger.Debugf("Getting projects with name %v", projectName)
var projectResponse struct {
TotalCount int `json:"totalCount"`
FilteredCount int `json:"filteredCount"`
Projects []Project `json:"projects"`
}
header := http.Header{}
header.Set("Accept-Type", "application/json")
var data []byte
var err error
body := url.Values{}
body.Add("name", projectName)
data, err = sendRequest(sys, http.MethodGet, fmt.Sprintf("/projects/?%v", body.Encode()), nil, header, []int{404})
if err != nil {
return []Project{}, errors.Wrapf(err, "fetching project %v failed", projectName)
}
err = json.Unmarshal(data, &projectResponse)
return projectResponse.Projects, err
}
// GetProjectsByNameAndGroup returns the project addressed by project name from the Checkmarx backend which the user has access to
// Updated for Cx1
func (sys *SystemInstance) GetProjectsByNameAndGroup(projectName, groupID string) ([]Project, error) {
sys.logger.Debugf("Getting projects with name %v of group %v...", projectName, groupID)
var projectResponse struct {
TotalCount int `json:"totalCount"`
FilteredCount int `json:"filteredCount"`
Projects []Project `json:"projects"`
}
header := http.Header{}
header.Set("Accept-Type", "application/json")
var data []byte
var err error
body := url.Values{}
if len(groupID) > 0 {
body.Add("groups", groupID)
}
if len(projectName) > 0 {
body.Add("name", projectName)
}
if len(body) > 0 {
data, err = sendRequest(sys, http.MethodGet, fmt.Sprintf("/projects/?%v", body.Encode()), nil, header, []int{404})
} else {
data, err = sendRequest(sys, http.MethodGet, "/projects/", nil, header, []int{404})
}
if err != nil {
return projectResponse.Projects, errors.Wrapf(err, "fetching project %v failed", projectName)
}
err = json.Unmarshal(data, &projectResponse)
return projectResponse.Projects, err
}
// CreateProject creates a new project in the Checkmarx backend
// Updated for Cx1
func (sys *SystemInstance) CreateProject(projectName string, groupIDs []string) (Project, error) {
var project Project
jsonData := map[string]interface{}{
"name": projectName,
"groups": groupIDs,
"origin": cxOrigin,
"criticality": 3, // default
// multiple additional parameters exist as options
}
jsonValue, err := json.Marshal(jsonData)
if err != nil {
return project, errors.Wrapf(err, "failed to marshal project data")
}
header := http.Header{}
header.Set("Content-Type", "application/json")
data, err := sendRequest(sys, http.MethodPost, "/projects", bytes.NewBuffer(jsonValue), header, []int{})
if err != nil {
return project, errors.Wrapf(err, "failed to create project %v", projectName)
}
err = json.Unmarshal(data, &project)
return project, err
}
func (sys *SystemInstance) CreateProjectInApplication(projectName, applicationID string, groupIDs []string) (Project, error) {
var project Project
jsonData := map[string]interface{}{
"name": projectName,
"groups": groupIDs,
"origin": cxOrigin,
"criticality": 3, // default
// multiple additional parameters exist as options
}
jsonValue, err := json.Marshal(jsonData)
if err != nil {
return project, errors.Wrapf(err, "failed to marshal project data")
}
header := http.Header{}
header.Set("Content-Type", "application/json")
data, err := sendRequest(sys, http.MethodPost, fmt.Sprintf("/projects/application/%v", applicationID), bytes.NewBuffer(jsonValue), header, []int{})
if err != nil {
return project, errors.Wrapf(err, "failed to create project %v under %v", projectName, applicationID)
}
err = json.Unmarshal(data, &project)
return project, err
}
// New for Cx1
func (sys *SystemInstance) GetUploadURI() (string, error) {
sys.logger.Debug("Retrieving upload URI")
header := http.Header{}
header.Set("Content-Type", "application/json")
resp, err := sendRequest(sys, http.MethodPost, "/uploads", nil, header, []int{})
if err != nil {
return "", errors.Wrap(err, "failed to get an upload uri")
}
responseData := make(map[string]string)
json.Unmarshal(resp, &responseData)
sys.logger.Debugf("Upload URI %s", responseData["url"])
return responseData["url"], nil
}
func (sys *SystemInstance) UploadProjectSourceCode(projectID string, zipFile string) (string, error) {
sys.logger.Debugf("Preparing to upload file %v...", zipFile)
// get URI
uploadUri, err := sys.GetUploadURI()
if err != nil {
return "", err
}
header := http.Header{}
header.Add("Accept-Encoding", "gzip,deflate")
header.Add("Content-Type", "application/zip")
header.Add("Accept", "application/json")
zipContents, err := os.ReadFile(zipFile)
if err != nil {
sys.logger.Error("Failed to Read the File " + zipFile + ": " + err.Error())
return "", err
}
response, err := sendRequestInternal(sys, http.MethodPut, uploadUri, bytes.NewReader(zipContents), header, []int{})
if err != nil {
sys.logger.Errorf("Failed to upload file %v: %s", zipFile, err)
return uploadUri, err
}
sys.logger.Debugf("Upload request response: %v", string(response))
return uploadUri, nil
}
func (sys *SystemInstance) scanProject(scanConfig map[string]interface{}) (Scan, error) {
scan := Scan{}
jsonValue, err := json.Marshal(scanConfig)
header := http.Header{}
header.Set("Content-Type", "application/json")
sys.logger.Tracef("Starting scan with settings: " + string(jsonValue))
data, err := sendRequest(sys, http.MethodPost, "/scans/", bytes.NewBuffer(jsonValue), header, []int{})
if err != nil {
return scan, err
}
err = json.Unmarshal(data, &scan)
return scan, err
}
func (sys *SystemInstance) ScanProjectZip(projectID, sourceUrl, branch string, settings []ScanConfiguration) (Scan, error) {
jsonBody := map[string]interface{}{
"project": map[string]interface{}{"id": projectID},
"type": "upload",
"handler": map[string]interface{}{
"uploadurl": sourceUrl,
"branch": branch,
},
"config": settings,
}
scan, err := sys.scanProject(jsonBody)
if err != nil {
return scan, errors.Wrapf(err, "Failed to start a zip scan for project %v", projectID)
}
return scan, err
}
func (sys *SystemInstance) ScanProjectGit(projectID, repoUrl, branch string, settings []ScanConfiguration) (Scan, error) {
jsonBody := map[string]interface{}{
"project": map[string]interface{}{"id": projectID},
"type": "git",
"handler": map[string]interface{}{
"repoUrl": repoUrl,
"branch": branch,
},
"config": settings,
}
scan, err := sys.scanProject(jsonBody)
if err != nil {
return scan, errors.Wrapf(err, "Failed to start a git scan for project %v", projectID)
}
return scan, err
}
func (sys *SystemInstance) ScanProject(projectID, sourceUrl, branch, scanType string, settings []ScanConfiguration) (Scan, error) {
if scanType == "upload" {
return sys.ScanProjectZip(projectID, sourceUrl, branch, settings)
} else if scanType == "git" {
return sys.ScanProjectGit(projectID, sourceUrl, branch, settings)
}
return Scan{}, errors.New("Invalid scanType provided, must be 'upload' or 'git'")
}
//func (sys *SystemInstance) UpdateProjectExcludeSettings(projectID string, excludeFolders string, excludeFiles string) error {
// replaced by SetProjectFileFilter
// Updated for Cx1: GetPresets loads the preset values defined in the Checkmarx backend
func (sys *SystemInstance) GetPresets() ([]Preset, error) {
sys.logger.Debug("Getting Presets...")
var presets []Preset
data, err := sendRequest(sys, http.MethodGet, "/queries/presets", nil, http.Header{}, []int{})
if err != nil {
sys.logger.Errorf("Fetching presets failed: %s", err)
return presets, err
}
err = json.Unmarshal(data, &presets)
return presets, err
}
// New for Cx1
func (sys *SystemInstance) GetProjectConfiguration(projectID string) ([]ProjectConfigurationSetting, error) {
sys.logger.Debug("Getting project configuration")
var projectConfigurations []ProjectConfigurationSetting
params := url.Values{
"project-id": {projectID},
}
data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/configuration/project?%v", params.Encode()), nil, http.Header{}, []int{})
if err != nil {
sys.logger.Errorf("Failed to get project configuration for project ID %v: %s", projectID, err)
return projectConfigurations, err
}
err = json.Unmarshal(data, &projectConfigurations)
return projectConfigurations, err
}
// UpdateProjectConfiguration updates the configuration of the project addressed by projectID
// Updated for Cx1
func (sys *SystemInstance) UpdateProjectConfiguration(projectID string, settings []ProjectConfigurationSetting) error {
if len(settings) == 0 {
return errors.New("Empty list of settings provided.")
}
params := url.Values{
"project-id": {projectID},
}
jsonValue, err := json.Marshal(settings)
if err != nil {
sys.logger.Errorf("Failed to marshal settings.")
return err
}
_, err = sendRequest(sys, http.MethodPatch, fmt.Sprintf("/configuration/project?%v", params.Encode()), bytes.NewReader(jsonValue), http.Header{}, []int{})
if err != nil {
sys.logger.Errorf("Failed to update project configuration: %s", err)
return err
}
return nil
}
func (sys *SystemInstance) SetProjectBranch(projectID, branch string, allowOverride bool) error {
var setting ProjectConfigurationSetting
setting.Key = "scan.handler.git.branch"
setting.Value = branch
setting.AllowOverride = allowOverride
return sys.UpdateProjectConfiguration(projectID, []ProjectConfigurationSetting{setting})
}
func (sys *SystemInstance) SetProjectPreset(projectID, presetName string, allowOverride bool) error {
var setting ProjectConfigurationSetting
setting.Key = "scan.config.sast.presetName"
setting.Value = presetName
setting.AllowOverride = allowOverride
return sys.UpdateProjectConfiguration(projectID, []ProjectConfigurationSetting{setting})
}
func (sys *SystemInstance) SetProjectLanguageMode(projectID, languageMode string, allowOverride bool) error {
var setting ProjectConfigurationSetting
setting.Key = "scan.config.sast.languageMode"
setting.Value = languageMode
setting.AllowOverride = allowOverride
return sys.UpdateProjectConfiguration(projectID, []ProjectConfigurationSetting{setting})
}
func (sys *SystemInstance) SetProjectFileFilter(projectID, filter string, allowOverride bool) error {
var setting ProjectConfigurationSetting
setting.Key = "scan.config.sast.filter"
setting.Value = filter
setting.AllowOverride = allowOverride
// TODO - apply the filter across all languages? set up separate calls per engine? engine as param?
return sys.UpdateProjectConfiguration(projectID, []ProjectConfigurationSetting{setting})
}
// GetScans returns all scan status on the project addressed by projectID
func (sys *SystemInstance) GetScan(scanID string) (Scan, error) {
var scan Scan
data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/scans/%v", scanID), nil, http.Header{}, []int{})
if err != nil {
sys.logger.Errorf("Failed to fetch scan with ID %v: %s", scanID, err)
return scan, errors.Wrapf(err, "failed to fetch scan with ID %v", scanID)
}
json.Unmarshal(data, &scan)
return scan, nil
}
func (sys *SystemInstance) GetScanMetadata(scanID string) (ScanMetadata, error) {
var scanmeta ScanMetadata
data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/sast-metadata/%v", scanID), nil, http.Header{}, []int{})
if err != nil {
sys.logger.Errorf("Failed to fetch metadata for scan with ID %v: %s", scanID, err)
return scanmeta, errors.Wrapf(err, "failed to fetch metadata for scan with ID %v", scanID)
}
json.Unmarshal(data, &scanmeta)
return scanmeta, nil
}
// GetScans returns all scan status on the project addressed by projectID
func (sys *SystemInstance) GetScanWorkflow(scanID string) ([]WorkflowLog, error) {
var workflow []WorkflowLog
data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/scans/%v/workflow", scanID), nil, http.Header{}, []int{})
if err != nil {
sys.logger.Errorf("Failed to fetch scan with ID %v: %s", scanID, err)
return []WorkflowLog{}, errors.Wrapf(err, "failed to fetch scan with ID %v", scanID)
}
json.Unmarshal(data, &workflow)
return workflow, nil
}
// GetScans returns all scan status on the project addressed by projectID
func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, error) {
var scanResponse struct {
TotalCount uint64
FilteredTotalCount uint64
Scans []Scan
}
body := url.Values{
"project-id": {projectID},
"offset": {fmt.Sprintf("%v", 0)},
"limit": {fmt.Sprintf("%v", limit)},
"sort": {"+created_at"},
}
header := http.Header{}
header.Set("Accept-Type", "application/json")
data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/scans/?%v", body.Encode()), nil, header, []int{})
if err != nil {
sys.logger.Errorf("Failed to fetch scans of project %v: %s", projectID, err)
return []Scan{}, errors.Wrapf(err, "failed to fetch scans of project %v", projectID)
}
err = json.Unmarshal(data, &scanResponse)
return scanResponse.Scans, err
}
func (sys *SystemInstance) GetLastScansByStatus(projectID string, limit int, status []string) ([]Scan, error) {
var scanResponse struct {
TotalCount uint64
FilteredTotalCount uint64
Scans []Scan
}
body := url.Values{
"project-id": {projectID},
"offset": {fmt.Sprintf("%d", 0)},
"limit": {fmt.Sprintf("%d", limit)},
"sort": {"+created_at"},
"statuses": status,
}
data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/scans/?%v", body.Encode()), nil, nil, []int{})
if err != nil {
sys.logger.Errorf("Failed to fetch scans of project %v: %s", projectID, err)
return []Scan{}, errors.Wrapf(err, "failed to fetch scans of project %v", projectID)
}
err = json.Unmarshal(data, &scanResponse)
return scanResponse.Scans, err
}
func (s *Scan) IsIncremental() (bool, error) {
for _, scanconfig := range s.Metadata.Configs {
if scanconfig.ScanType == "sast" {
if val, ok := scanconfig.Values["incremental"]; ok {
return val == "true", nil
}
}
}
return false, errors.New(fmt.Sprintf("Scan %v did not have a sast-engine incremental flag set", s.ScanID))
}
func (sys *SystemInstance) GetScanResults(scanID string, limit uint64) ([]ScanResult, error) {
sys.logger.Debug("Get Cx1 Scan Results")
var resultResponse struct {
Results []ScanResult
TotalCount int
}
params := url.Values{
"scan-id": {scanID},
"limit": {fmt.Sprintf("%d", limit)},
"state": []string{},
"severity": []string{},
"status": []string{},
}
response, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/results/?%v", params.Encode()), nil, nil, []int{})
if err != nil && len(response) == 0 {
sys.logger.Errorf("Failed to retrieve scan results for scan ID %v", scanID)
return []ScanResult{}, err
}
err = json.Unmarshal(response, &resultResponse)
if err != nil {
sys.logger.Errorf("Failed while parsing response: %s", err)
sys.logger.Tracef("Response contents: %s", string(response))
return []ScanResult{}, err
}
sys.logger.Debugf("Retrieved %d results", resultResponse.TotalCount)
if len(resultResponse.Results) != resultResponse.TotalCount {
sys.logger.Warnf("Expected results total count %d but parsed only %d", resultResponse.TotalCount, len(resultResponse.Results))
sys.logger.Warnf("Response was: %v", string(response))
}
return resultResponse.Results, nil
}
func (s *ScanSummary) TotalCount() uint64 {
var count uint64
count = 0
for _, c := range s.SASTCounters.StateCounters {
count += c.Counter
}
return count
}
func (sys *SystemInstance) GetScanSummary(scanID string) (ScanSummary, error) {
var ScansSummaries struct {
ScanSum []ScanSummary `json:"scansSummaries"`
TotalCount uint64
}
params := url.Values{
"scan-ids": {scanID},
"include-queries": {"false"},
"include-status-counters": {"true"},
"include-files": {"false"},
}
data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/scan-summary/?%v", params.Encode()), nil, nil, []int{})
if err != nil {
sys.logger.Errorf("Failed to fetch metadata for scan with ID %v: %s", scanID, err)
return ScanSummary{}, errors.Wrapf(err, "failed to fetch metadata for scan with ID %v", scanID)
}
err = json.Unmarshal(data, &ScansSummaries)
if err != nil {
return ScanSummary{}, err
}
if ScansSummaries.TotalCount == 0 {
return ScanSummary{}, errors.New(fmt.Sprintf("Failed to retrieve scan summary for scan ID %v", scanID))
}
if len(ScansSummaries.ScanSum) == 0 {
sys.logger.Errorf("Failed to parse data, 0-len ScanSum.\n%v", string(data))
return ScanSummary{}, errors.New("Fail")
}
return ScansSummaries.ScanSum[0], nil
}
func (sys *SystemInstance) GetResultsPredicates(SimilarityID int64, ProjectID string) ([]ResultsPredicates, error) {
sys.logger.Debugf("Fetching results predicates for project %v similarityId %d", ProjectID, SimilarityID)
var Predicates struct {
PredicateHistoryPerProject []struct {
ProjectID string
SimilarityID int64 `json:"similarityId,string"`
Predicates []ResultsPredicates
TotalCount uint
}
TotalCount uint
}
response, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/sast-results-predicates/%d?project-ids=%v", SimilarityID, ProjectID), nil, nil, []int{})
if err != nil {
return []ResultsPredicates{}, err
}
err = json.Unmarshal(response, &Predicates)
if err != nil {
return []ResultsPredicates{}, err
}
if Predicates.TotalCount == 0 {
return []ResultsPredicates{}, nil
}
return Predicates.PredicateHistoryPerProject[0].Predicates, err
}
// RequestNewReport triggers the generation of a report for a specific scan addressed by scanID
func (sys *SystemInstance) RequestNewReport(scanID, projectID, branch, reportType string) (string, error) {
jsonData := map[string]interface{}{
"fileFormat": reportType,
"reportType": "ui",
"reportName": "scan-report",
"data": map[string]interface{}{
"scanId": scanID,
"projectId": projectID,
"branchName": branch,
"sections": []string{
"ScanSummary",
"ExecutiveSummary",
"ScanResults",
},
"scanners": []string{"SAST"},
"host": "",
},
}
jsonValue, _ := json.Marshal(jsonData)
header := http.Header{}
header.Set("cxOrigin", cxOrigin)
header.Set("Content-Type", "application/json")
data, err := sendRequest(sys, http.MethodPost, "/reports", bytes.NewBuffer(jsonValue), header, []int{})
if err != nil {
return "", errors.Wrapf(err, "Failed to trigger report generation for scan %v", scanID)
} else {
sys.logger.Infof("Generating report %v", string(data))
}
var reportResponse struct {
ReportId string
}
err = json.Unmarshal(data, &reportResponse)
return reportResponse.ReportId, err
}
// GetReportStatus returns the status of the report generation process
func (sys *SystemInstance) GetReportStatus(reportID string) (ReportStatus, error) {
var response ReportStatus
header := http.Header{}
header.Set("Accept", "application/json")
data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/reports/%v", reportID), nil, header, []int{})
if err != nil {
sys.logger.Errorf("Failed to fetch report status for reportID %v: %s", reportID, err)
return response, errors.Wrapf(err, "failed to fetch report status for reportID %v", reportID)
}
json.Unmarshal(data, &response)
return response, nil
}
func (sys *SystemInstance) GetQueries() ([]Query, error) {
sys.logger.Debug("Get Cx1 Queries")
var queries []Query
response, err := sendRequest(sys, http.MethodGet, "/presets/queries", nil, nil, []int{})
if err != nil {
return queries, err
}
err = json.Unmarshal(response, &queries)
if err != nil {
sys.logger.Errorf("Failed to parse %v", string(response))
}
return queries, err
}
func shortenGUID(guid string) string {
return fmt.Sprintf("%v..%v", guid[:2], guid[len(guid)-2:])
}
// DownloadReport downloads the report addressed by reportID and returns the XML contents
func (sys *SystemInstance) DownloadReport(reportUrl string) ([]byte, error) {
header := http.Header{}
header.Set("Accept", "application/json")
data, err := sendRequestInternal(sys, http.MethodGet, reportUrl, nil, header, []int{})
if err != nil {
return []byte{}, errors.Wrapf(err, "failed to download report from url: %v", reportUrl)
}
return data, nil
}