Cxone release supporting applications (#4548)

* Initial in progress

* compiling but not yet functional

* Missed file

* updated checkmarxone step

* Working up to fetching a project then breaks

* Missed file

* Breaks when retrieving projects+proxy set

* Create project & run scan working, now polling

* Fixed polling

* added back the zipfile remove command

* Fixed polling again

* Generates and downloads PDF report

* Updated and working, prep for refactor

* Added compliance steps

* Cleanup, reporting, added groovy connector

* fixed groovy file

* checkmarxone to checkmarxOne

* checkmarxone to checkmarxOne

* split credentials (id+secret, apikey), renamed pullrequestname to branch, groovy fix

* Fixed filenames & yaml

* missed the metadata_generated.go

* added json to sarif conversion

* fix:type in new checkmarxone package

* fix:type in new checkmarxone package

* removed test logs, added temp error log for creds

* extra debugging to fix crash

* improved auth logging, fixed query parse issue

* fixed bug with group fetch when using oauth user

* CWE can be -1 if not defined, can't be uint

* Query also had CweID

* Disabled predicates-fetch in sarif generation

* Removing leftover info log message

* Better error handling

* fixed default preset configuration

* removing .bat files - sorry

* Cleanup per initial review

* refactoring per Gist, fixed project find, add apps

* small fix - sorry for commit noise while testing

* Fixing issues with incremental scans.

* removing maxretries

* Updated per PR feedback, further changes todo toda

* JSON Report changes and reporting cleanup

* removing .bat (again?)

* adding docs, groovy unit test, linter fixes

* Started adding tests maybe 15% covered

* fix(checkmarxOne): test cases for pkg and reporting

* fix(checkmarxOne):fix formatting

* feat(checkmarxone): update interface with missing method

* feat(checkmarxone):change runStep signature to be able to inject dependency

* feat(checkmarxone): add tests for step (wip)

* Adding a bit more coverage

* feat(checkmarxOne): fix code review

* feat(checkmarxOne): fix code review

* feat(checkmarxOne): fix code review

* feat(checkmarxOne): fix integration test PR

* adding scan-summary bug workaround, reportgen fail

* enforceThresholds fix when no results passed in

* fixed gap when preset empty in yaml & project conf

* fixed another gap in preset selection

* fix 0-result panic

* fail when no preset is set anywhere

* removed comment

* initial project-under-app support

* fixing sarif reportgen

* some cleanup of error messages

* post-merge test fixes

* revert previous upstream merge

* fix:formatting

* fix(checkmarxOne):yamllint too many blank lines

* fix(checkmarxOne):unit test

* fix(checkmarxOne):generated code

---------

Co-authored-by: thtri <trinhthanhhai@gmail.com>
Co-authored-by: Thanh-Hai Trinh <thanh.hai.trinh@sap.com>

cmd/checkmarxOneExecuteScan.go

@@ -80,18 +80,31 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS
 
 	cx1sh.Group, err = cx1sh.GetGroup() // used when creating a project and when generating a SARIF report
 	if err != nil {
-		return fmt.Errorf("failed to get group: %s", err)
+		log.Entry().WithError(err).Warnf("failed to get group")
 	}
 
 	if cx1sh.Project == nil {
 		cx1sh.App, err = cx1sh.GetApplication() // read application name from piper config (optional) and get ID from CxONE API
 		if err != nil {
-			log.Entry().WithError(err).Warnf("failed to get application")
+			log.Entry().WithError(err).Warnf("Failed to get application - will attempt to create the project on the Tenant level")
 		}
 		cx1sh.Project, err = cx1sh.CreateProject() // requires groups, repoUrl, mainBranch, origin, tags, criticality
 		if err != nil {
 			return fmt.Errorf("failed to create project: %s", err)
 		}
+	} else {
+		cx1sh.Project, err = cx1sh.GetProjectByID(cx1sh.Project.ProjectID)
+		if err != nil {
+			return fmt.Errorf("failed to get project by ID: %s", err)
+		} else {
+			if len(cx1sh.Project.Applications) > 0 {
+				appId := cx1sh.Project.Applications[0]
+				cx1sh.App, err = cx1sh.GetApplicationByID(cx1sh.Project.Applications[0])
+				if err != nil {
+					return fmt.Errorf("failed to retrieve information for project's assigned application %v", appId)
+				}
+			}
+		}
 	}
 
 	err = cx1sh.SetProjectPreset()
@@ -202,6 +215,11 @@ func (c *checkmarxOneExecuteScanHelper) GetProjectByName() (*checkmarxOne.Projec
 	return nil, fmt.Errorf("project not found")
 }
 
+func (c *checkmarxOneExecuteScanHelper) GetProjectByID(projectId string) (*checkmarxOne.Project, error) {
+	project, err := c.sys.GetProjectByID(projectId)
+	return &project, err
+}
+
 func (c *checkmarxOneExecuteScanHelper) GetGroup() (*checkmarxOne.Group, error) {
 	if len(c.config.GroupName) > 0 {
 		group, err := c.sys.GetGroupByName(c.config.GroupName)
@@ -210,8 +228,7 @@ func (c *checkmarxOneExecuteScanHelper) GetGroup() (*checkmarxOne.Group, error)
 		}
 		return &group, nil
 	}
-
-	return nil, fmt.Errorf("No group ID or group name provided")
+	return nil, fmt.Errorf("No group name specified in configuration")
 }
 
 func (c *checkmarxOneExecuteScanHelper) GetApplication() (*checkmarxOne.Application, error) {
@@ -223,7 +240,16 @@ func (c *checkmarxOneExecuteScanHelper) GetApplication() (*checkmarxOne.Applicat
 
 		return &app, nil
 	}
-	return nil, fmt.Errorf("No application named %v found", c.config.ApplicationName)
+	return nil, fmt.Errorf("No application name specified in configuration")
+}
+
+func (c *checkmarxOneExecuteScanHelper) GetApplicationByID(applicationId string) (*checkmarxOne.Application, error) {
+	app, err := c.sys.GetApplicationByID(applicationId)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get Checkmarx One application by Name %v: %s", c.config.ApplicationName, err)
+	}
+
+	return &app, nil
 }
 
 func (c *checkmarxOneExecuteScanHelper) CreateProject() (*checkmarxOne.Project, error) {
@@ -231,7 +257,19 @@ func (c *checkmarxOneExecuteScanHelper) CreateProject() (*checkmarxOne.Project,
 		return nil, fmt.Errorf("Preset is required to create a project")
 	}
 
-	project, err := c.sys.CreateProject(c.config.ProjectName, []string{c.Group.GroupID})
+	var project checkmarxOne.Project
+	var err error
+	var groupIDs []string = []string{}
+	if c.Group != nil {
+		groupIDs = []string{c.Group.GroupID}
+	}
+
+	if c.App != nil {
+		project, err = c.sys.CreateProjectInApplication(c.config.ProjectName, c.App.ApplicationID, groupIDs)
+	} else {
+		project, err = c.sys.CreateProject(c.config.ProjectName, groupIDs)
+	}
+
 	if err != nil {
 		return nil, fmt.Errorf("Error when trying to create project: %s", err)
 	}
@@ -648,8 +686,18 @@ func (c *checkmarxOneExecuteScanHelper) getDetailedResults(scan *checkmarxOne.Sc
 	resultMap["ScanId"] = scan.ScanID
 	resultMap["ProjectId"] = c.Project.ProjectID
 	resultMap["ProjectName"] = c.Project.Name
-	resultMap["Group"] = c.Group.GroupID
-	resultMap["GroupFullPathOnReportDate"] = c.Group.Name
+
+	resultMap["Group"] = ""
+	resultMap["GroupFullPathOnReportDate"] = ""
+
+	if c.App != nil {
+		resultMap["Application"] = c.App.ApplicationID
+		resultMap["ApplicationFullPathOnReportDate"] = c.App.Name
+	} else {
+		resultMap["Application"] = ""
+		resultMap["ApplicationFullPathOnReportDate"] = ""
+	}
+
 	resultMap["ScanStart"] = scan.CreatedAt
 
 	scanCreated, err := time.Parse(time.RFC3339, scan.CreatedAt)
@@ -669,7 +717,6 @@ func (c *checkmarxOneExecuteScanHelper) getDetailedResults(scan *checkmarxOne.Sc
 
 	resultMap["LinesOfCodeScanned"] = scanmeta.LOC
 	resultMap["FilesScanned"] = scanmeta.FileCount
-
 	resultMap["ToolVersion"] = "Cx1 Gap: No API for this"
 
 	if scanmeta.IsIncremental {
@@ -1077,6 +1124,7 @@ func (c *checkmarxOneExecuteScanHelper) reportToInflux(results *map[string]inter
 	c.influx.checkmarxOne_data.fields.lines_of_code_scanned = (*results)["LinesOfCodeScanned"].(int)
 	c.influx.checkmarxOne_data.fields.files_scanned = (*results)["FilesScanned"].(int)
 	c.influx.checkmarxOne_data.fields.tool_version = (*results)["ToolVersion"].(string)
+
 	c.influx.checkmarxOne_data.fields.scan_type = (*results)["ScanType"].(string)
 	c.influx.checkmarxOne_data.fields.preset = (*results)["Preset"].(string)
 	c.influx.checkmarxOne_data.fields.deep_link = (*results)["DeepLink"].(string)

cmd/checkmarxOneExecuteScan_generated.go

@@ -369,7 +369,7 @@ func addCheckmarxOneExecuteScanFlags(cmd *cobra.Command, stepConfig *checkmarxOn
 	cmd.Flags().StringVar(&stepConfig.IamURL, "iamUrl", os.Getenv("PIPER_iamUrl"), "The URL pointing to the access control root of the checkmarxOne IAM server to be used")
 	cmd.Flags().StringVar(&stepConfig.Tenant, "tenant", os.Getenv("PIPER_tenant"), "The name of the checkmarxOne tenant to be used")
 	cmd.Flags().StringVar(&stepConfig.SourceEncoding, "sourceEncoding", `1`, "The source encoding to be used, if not set explicitly the project's default will be used  [Not yet supported]")
-	cmd.Flags().StringVar(&stepConfig.GroupName, "groupName", os.Getenv("PIPER_groupName"), "The full name of the group to assign newly created projects to which is preferred to groupId")
+	cmd.Flags().StringVar(&stepConfig.GroupName, "groupName", os.Getenv("PIPER_groupName"), "The full name of the group to which the newly created projects will be assigned")
 	cmd.Flags().StringVar(&stepConfig.ApplicationName, "applicationName", os.Getenv("PIPER_applicationName"), "The full name of the Checkmarx One application to which the newly created projects will be assigned")
 	cmd.Flags().StringVar(&stepConfig.ClientID, "clientId", os.Getenv("PIPER_clientId"), "The username to authenticate")
 	cmd.Flags().BoolVar(&stepConfig.VerifyOnly, "verifyOnly", false, "Whether the step shall only apply verification checks or whether it does a full scan and check cycle")

cmd/checkmarxOneExecuteScan_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
-	"github.com/SAP/jenkins-library/pkg/checkmarxone"
+	checkmarxOne "github.com/SAP/jenkins-library/pkg/checkmarxone"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
 	"github.com/pkg/errors"
 )
@@ -37,6 +37,10 @@ func (sys *checkmarxOneSystemMock) GetApplicationByName(appname string) (checkma
 	return checkmarxOne.Application{}, nil
 }
 
+func (sys *checkmarxOneSystemMock) GetApplicationByID(appname string) (checkmarxOne.Application, error) {
+	return checkmarxOne.Application{}, nil
+}
+
 func (sys *checkmarxOneSystemMock) UpdateApplication(app *checkmarxOne.Application) error {
 	return nil
 }
@@ -93,6 +97,10 @@ func (sys *checkmarxOneSystemMock) CreateProject(projectName string, groupIDs []
 	return checkmarxOne.Project{}, nil
 }
 
+func (sys *checkmarxOneSystemMock) CreateProjectInApplication(projectName, applicationId string, groupIDs []string) (checkmarxOne.Project, error) {
+	return checkmarxOne.Project{}, nil
+}
+
 func (sys *checkmarxOneSystemMock) GetPresets() ([]checkmarxOne.Preset, error) {
 	return []checkmarxOne.Preset{}, nil
 }
@@ -285,7 +293,7 @@ func TestGetGroup(t *testing.T) {
 
 		cx1sh := checkmarxOneExecuteScanHelper{nil, options, sys, nil, nil, nil, nil, nil, nil}
 		_, err := cx1sh.GetGroup()
-		assert.Contains(t, fmt.Sprint(err), "No group ID or group name provided")
+		assert.Contains(t, fmt.Sprint(err), "No group name specified in configuration")
 	})
 
 	t.Run("group name not found", func(t *testing.T) {

cmd/golangBuild.go

@@ -523,7 +523,6 @@ func runGolangBuildPerArchitecture(config *golangBuildOptions, goModFile *modfil
 }
 
 func runBOMCreation(utils golangBuildUtils, outputFilename string) error {
-
 	if err := utils.RunExecutable("cyclonedx-gomod", "mod", "-licenses", fmt.Sprintf("-verbose=%t", GeneralConfig.Verbose), "-test", "-output", outputFilename, "-output-version", "1.4"); err != nil {
 		return fmt.Errorf("BOM creation failed: %w", err)
 	}

pkg/checkmarxone/checkmarxone.go

@@ -62,16 +62,17 @@ type Preset struct {
 // Project - Project Structure
 // Updated for Cx1
 type Project struct {
-	ProjectID   string            `json:"id"`
-	Name        string            `json:"name"`
-	CreatedAt   string            `json:"createdAt"`
-	UpdatedAt   string            `json:"updatedAt"`
-	Groups      []string          `json:"groups"`
-	Tags        map[string]string `json:"tags"`
-	RepoUrl     string            `json:"repoUrl"`
-	MainBranch  string            `json:"mainBranch"`
-	Origin      string            `json:"origin"`
-	Criticality int               `json:"criticality"`
+	ProjectID    string            `json:"id"`
+	Name         string            `json:"name"`
+	CreatedAt    string            `json:"createdAt"`
+	UpdatedAt    string            `json:"updatedAt"`
+	Groups       []string          `json:"groups"`
+	Applications []string          `json:"applicationIds"`
+	Tags         map[string]string `json:"tags"`
+	RepoUrl      string            `json:"repoUrl"`
+	MainBranch   string            `json:"mainBranch"`
+	Origin       string            `json:"origin"`
+	Criticality  int               `json:"criticality"`
 }
 
 // New for Cx1
@@ -290,6 +291,7 @@ type System interface {
 
 	CreateApplication(appname string) (Application, error)
 	GetApplicationByName(appname string) (Application, error)
+	GetApplicationByID(appId string) (Application, error)
 	UpdateApplication(app *Application) error
 
 	GetScan(scanID string) (Scan, error)
@@ -307,6 +309,7 @@ type System interface {
 
 	UploadProjectSourceCode(projectID string, zipFile string) (string, error)
 	CreateProject(projectName string, groupIDs []string) (Project, error)
+	CreateProjectInApplication(projectName, applicationID string, groupIDs []string) (Project, error)
 	GetPresets() ([]Preset, error)
 	GetProjectByID(projectID string) (Project, error)
 	GetProjectsByName(projectName string) ([]Project, error)
@@ -538,6 +541,21 @@ func (sys *SystemInstance) GetApplicationsByName(name string, limit uint64) ([]A
 	return ApplicationResponse.Applications, err
 }
 
+func (sys *SystemInstance) GetApplicationByID(appId string) (Application, error) {
+	sys.logger.Debugf("Get Cx1 Application by ID: %v", appId)
+
+	var ret Application
+
+	response, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/applications/%v", appId), nil, nil, []int{})
+
+	if err != nil {
+		return ret, err
+	}
+
+	err = json.Unmarshal(response, &ret)
+	return ret, err
+}
+
 func (sys *SystemInstance) GetApplicationByName(name string) (Application, error) {
 	apps, err := sys.GetApplicationsByName(name, 0)
 	if err != nil {
@@ -796,6 +814,33 @@ func (sys *SystemInstance) CreateProject(projectName string, groupIDs []string)
 	return project, err
 }
 
+func (sys *SystemInstance) CreateProjectInApplication(projectName, applicationID string, groupIDs []string) (Project, error) {
+	var project Project
+	jsonData := map[string]interface{}{
+		"name":        projectName,
+		"groups":      groupIDs,
+		"origin":      cxOrigin,
+		"criticality": 3, // default
+		// multiple additional parameters exist as options
+	}
+
+	jsonValue, err := json.Marshal(jsonData)
+	if err != nil {
+		return project, errors.Wrapf(err, "failed to marshal project data")
+	}
+
+	header := http.Header{}
+	header.Set("Content-Type", "application/json")
+
+	data, err := sendRequest(sys, http.MethodPost, fmt.Sprintf("/projects/application/%v", applicationID), bytes.NewBuffer(jsonValue), header, []int{})
+	if err != nil {
+		return project, errors.Wrapf(err, "failed to create project %v under %v", projectName, applicationID)
+	}
+
+	err = json.Unmarshal(data, &project)
+	return project, err
+}
+
 // New for Cx1
 func (sys *SystemInstance) GetUploadURI() (string, error) {
 	sys.logger.Debug("Retrieving upload URI")

pkg/checkmarxone/checkmarxone_test.go

@@ -50,6 +50,7 @@ func (sm *senderMock) UploadFile(url, file, fieldName string, header http.Header
 	sm.urlCalled = url
 	sm.header = header
 	return &http.Response{StatusCode: sm.httpStatusCode, Body: io.NopCloser(bytes.NewReader([]byte(sm.responseBody)))}, nil
+
 }
 func (sm *senderMock) UploadRequest(method, url, file, fieldName string, header http.Header, cookies []*http.Cookie, uploadType string) (*http.Response, error) {
 	sm.httpMethod = http.MethodPost

pkg/checkmarxone/reporting.go

@@ -41,9 +41,9 @@ type Finding struct {
 }
 
 type LowPerQuery struct {
-	QueryName string `json:"name"`
-	Total     int    `json:"total"`
+	QueryName string `json:"query"`
 	Audited   int    `json:"audited"`
+	Total     int    `json:"total"`
 }
 
 func CreateCustomReport(data *map[string]interface{}, insecure, neutral []string) reporting.ScanReport {
@@ -137,16 +137,18 @@ func CreateCustomReport(data *map[string]interface{}, insecure, neutral []string
 
 func CreateJSONHeaderReport(data *map[string]interface{}) CheckmarxOneReportData {
 	checkmarxReportData := CheckmarxOneReportData{
-		ToolName:    `CheckmarxOne`,
-		ProjectName: fmt.Sprint((*data)["ProjectName"]),
-		GroupID:     fmt.Sprint((*data)["Group"]),
-		GroupName:   fmt.Sprint((*data)["GroupFullPathOnReportDate"]),
-		DeepLink:    fmt.Sprint((*data)["DeepLink"]),
-		Preset:      fmt.Sprint((*data)["Preset"]),
-		ToolVersion: fmt.Sprint((*data)["ToolVersion"]),
-		ScanType:    fmt.Sprint((*data)["ScanType"]),
-		ProjectID:   fmt.Sprint((*data)["ProjectId"]),
-		ScanID:      fmt.Sprint((*data)["ScanId"]),
+		ToolName:        `CheckmarxOne`,
+		ProjectName:     fmt.Sprint((*data)["ProjectName"]),
+		GroupID:         fmt.Sprint((*data)["Group"]),
+		GroupName:       fmt.Sprint((*data)["GroupFullPathOnReportDate"]),
+		ApplicationID:   fmt.Sprint((*data)["Application"]),
+		ApplicationName: fmt.Sprint((*data)["ApplicationFullPathOnReportDate"]),
+		DeepLink:        fmt.Sprint((*data)["DeepLink"]),
+		Preset:          fmt.Sprint((*data)["Preset"]),
+		ToolVersion:     fmt.Sprint((*data)["ToolVersion"]),
+		ScanType:        fmt.Sprint((*data)["ScanType"]),
+		ProjectID:       fmt.Sprint((*data)["ProjectId"]),
+		ScanID:          fmt.Sprint((*data)["ScanId"]),
 	}
 
 	findings := []Finding{}

pkg/checkmarxone/reporting_test.go

@@ -12,6 +12,8 @@ func TestCreateJSONReport(t *testing.T) {
 	resultMap["ProjectName"] = `ssba`
 	resultMap["Group"] = `test-group`
 	resultMap["GroupFullPathOnReportDate"] = `test-group-path`
+	resultMap["Application"] = `test-app`
+	resultMap["ApplicationFullPathOnReportDate"] = `test-app-path`
 	resultMap["DeepLink"] = `https://cx1.sap/projects/f5702f86-b396-417f-82e2-4949a55d5382/scans?branch=master&page=1&id=21e40b36-0dd7-48e5-9768-da1a8f36c907`
 	resultMap["Preset"] = `Checkmarx Default`
 	resultMap["ToolVersion"] = `v1`

resources/metadata/checkmarxOneExecuteScan.yaml

@@ -256,7 +256,7 @@ spec:
         default: "1"
       - name: groupName
         type: string
-        description: The full name of the group to assign newly created projects to which is preferred to groupId
+        description: The full name of the group to which the newly created projects will be assigned
         scope:
           - PARAMETERS
           - STAGES