jenkins/sap-jenkins-library
1
0
Fork 0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-03-03 15:02:35 +02:00
Code Issues Releases Activity

Updates to toolrecord framework (#2986)

Browse Source 
* Toolrecord framework -
provide a common entry point for post processing code scan results

Changes to be committed:
	new file:   pkg/toolrecord/REAMDE_toolrecord.md
	new file:   pkg/toolrecord/toolrecord_main.go
	new file:   pkg/toolrecord/toolrecord_test.go

* Add toolrecord file to Checkmarx results
modified:   cmd/checkmarxExecuteScan.go

* Add toolrecord file to Fortify results
	modified:   cmd/fortifyExecuteScan.go

* Add toolrecord file to Whitesource results
modified:   cmd/whitesourceExecuteScan.go

* unset umask (#2927)

* (feat) adds error logging output for downloading reports from whitesource (#2928)

* Add toolrecord file to Protecode results

* address code climate findings (1/2)

* address codeclimate findings (2/2)

* add comments to all methods

* Toolrecord library:
- move all toolrun files into a subdirectory
- fix timestamp generation in filenames

* add protecode group's URL to toolrecord data

* fix syntax error from previous commit in cmd/protecodeExecuteScan.go

* toolrecord: fix projectVersionID and generated URLs in fortifyExecuteScan.go

* cmd/fortifyExecuteScan.go: replace a hard-coded servername with
config.ServerURL

* update description

* add toolrecord file to detectExecuteScan

* toolrecord/whitesource: add project names as context

Co-authored-by: Kevin Stiehl <kevin.stiehl@numericas.de>
Co-authored-by: ffeldmann <felix@bnbit.de>
Co-authored-by: Sven Merk <33895725+nevskrem@users.noreply.github.com>
This commit is contained in:
larsbrueckner 2021-07-23 08:48:48 +02:00 committed by GitHub
parent 3c41788405
commit dbbbe1f0b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 55 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
cmd/detectExecuteScan.go
View File

@ -14,6 +14,7 @@ import (
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/SAP/jenkins-library/pkg/telemetry"
"github.com/SAP/jenkins-library/pkg/toolrecord"
"github.com/SAP/jenkins-library/pkg/versioning"
)
@ -73,6 +74,13 @@ func detectExecuteScan(config detectExecuteScanOptions, _ *telemetry.CustomData)
WithError(err).
Fatal("failed to execute detect scan")
}
// create Toolrecord file
toolRecordFileName, err := createToolRecordDetect("./", config)
if err != nil {
// do not fail until the framework is well established
log.Entry().Warning("TR_DETECT: Failed to create toolrecord file "+toolRecordFileName, err)
}
}
func runDetect(config detectExecuteScanOptions, utils detectUtils) error {
@ -222,3 +230,26 @@ func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectU
return args, nil
}
// create toolrecord file for detect
//
//
func createToolRecordDetect(workspace string, config detectExecuteScanOptions) (string, error) {
record := toolrecord.New(workspace, "detectExecute", config.ServerURL)
projectId := "" // todo needs more research; according to synopsis documentation
productURL := "" // relevant ids can be found in the logfile
err := record.AddKeyData("project",
projectId,
config.ProjectName,
productURL)
if err != nil {
return "", err
}
record.AddContext("DetectTools", config.DetectTools)
err = record.Persist()
if err != nil {
return "", err
}
return record.GetFileName(), nil
}

10
cmd/fortifyExecuteScan.go
View File

@ -176,7 +176,7 @@ func runFortifyScan(config fortifyExecuteScanOptions, sys fortify.System, utils
// create toolrecord file
// tbd - how to handle verifyOnly
toolRecordFileName, err := createToolRecordFortify("./", config, project.ID, fortifyProjectName, fortifyProjectVersion)
toolRecordFileName, err := createToolRecordFortify("./", config, project.ID, fortifyProjectName, projectVersion.ID, fortifyProjectVersion)
if err != nil {
// do not fail until the framework is well established
log.Entry().Warning("TR_FORTIFY: Failed to create toolrecord file ...", err)
@ -259,7 +259,7 @@ func verifyFFProjectCompliance(config fortifyExecuteScanOptions, sys fortify.Sys
reports := []piperutils.Path{}
// Generate report
if config.Reporting {
resultURL := []byte(fmt.Sprintf("https://fortify.tools.sap/ssc/html/ssc/version/%v/fix/null/", projectVersion.ID))
resultURL := []byte(fmt.Sprintf("%v/html/ssc/version/%v/fix/null/", config.ServerURL, projectVersion.ID))
ioutil.WriteFile(fmt.Sprintf("%vtarget/%v-%v.%v", config.ModulePath, *project.Name, *projectVersion.Name, "txt"), resultURL, 0700)
data, err := generateAndDownloadQGateReport(config, sys, project, projectVersion)
@ -1017,7 +1017,7 @@ func getSeparator() string {
return ":"
}
func createToolRecordFortify(workspace string, config fortifyExecuteScanOptions, projectID int64, projectName, projectVersion string) (string, error) {
func createToolRecordFortify(workspace string, config fortifyExecuteScanOptions, projectID int64, projectName string, projectVersionID int64, projectVersion string) (string, error) {
record := toolrecord.New(workspace, "fortify", config.ServerURL)
// Project
err := record.AddKeyData("project",
@ -1028,9 +1028,9 @@ func createToolRecordFortify(workspace string, config fortifyExecuteScanOptions,
return "", err
}
// projectVersion
projectVersionURL := config.ServerURL + "/ssc/html/ssc/version/" + projectVersion
projectVersionURL := config.ServerURL + "/html/ssc/version/" + strconv.FormatInt(projectVersionID, 10)
err = record.AddKeyData("projectVersion",
projectVersion,
strconv.FormatInt(projectVersionID, 10),
projectVersion,
projectVersionURL)
if err != nil {

7
cmd/protecodeExecuteScan.go
View File

@ -375,16 +375,17 @@ func getTarName(config *protecodeExecuteScanOptions) string {
// todo: check if group and product names can be retrieved
func createToolRecordProtecode(workspace string, config *protecodeExecuteScanOptions, productID int, webuiURL string) (string, error) {
record := toolrecord.New(workspace, "protecode", config.ServerURL)
groupURL := config.ServerURL + "/#/groups/" + config.Group
err := record.AddKeyData("group",
config.Group,
config.Group,
"")
config.Group, // todo figure out display name
groupURL)
if err != nil {
return "", err
}
err = record.AddKeyData("product",
strconv.Itoa(productID),
strconv.Itoa(productID),
strconv.Itoa(productID), // todo figure out display name
webuiURL)
if err != nil {
return "", err

6
cmd/whitesourceExecuteScan.go
View File

@ -250,7 +250,7 @@ func checkAndReportScanResults(config *ScanOptions, scan *ws.Scan, utils whiteso
// create toolrecord file
// tbd - how to handle verifyOnly
toolRecordFileName, err := createToolRecordWhitesource("./", config)
toolRecordFileName, err := createToolRecordWhitesource("./", config, scan)
if err != nil {
// do not fail until the framework is well established
log.Entry().Warning("TR_WHITESOURCE: Failed to create toolrecord file ...", err)
@ -929,7 +929,7 @@ func persistScannedProjects(config *ScanOptions, scan *ws.Scan, commonPipelineEn
// Limitation: as the toolrecord file is designed to point to one scan result this generate a pointer
// to the product only, and not to the scanned projects
//
func createToolRecordWhitesource(workspace string, config *whitesourceExecuteScanOptions) (string, error) {
func createToolRecordWhitesource(workspace string, config *whitesourceExecuteScanOptions, scan *ws.Scan) (string, error) {
record := toolrecord.New(workspace, "whitesource", config.ServiceURL)
productURL := config.ServiceURL + "/Wss/WSS.html#!product;token=" + config.ProductToken
err := record.AddKeyData("product",
@ -939,6 +939,8 @@ func createToolRecordWhitesource(workspace string, config *whitesourceExecuteSca
if err != nil {
return "", err
}
record.AddContext("scannedProjects", scan.ScannedProjectNames)
record.AddContext("configuredProject", config.ProjectName+" - "+config.Version)
err = record.Persist()
if err != nil {
return "", err

2
pkg/toolrecord/REAMDE_toolrecord.md
View File

@ -1,4 +1,4 @@
Purpose of the "toolrecord" feature is to provide a common result file for all code scanners that allows consumers of the piper result files to locate the results in the respective tool backends for further reporting and post processing
Purpose of the "toolrecord" feature is to provide a common result file for tools (e.g. code scanners) to allow consumers of the piper result files to locate the results in the respective tool backends for further reporting and post processing
Currently it contains the minimal information to detect which tools have been executed, and where to locate the results in the respective tool backends.

14
pkg/toolrecord/toolrecord_main.go
View File

@ -5,8 +5,8 @@ import (
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
)
@ -56,9 +56,9 @@ func New(workspace, toolName, toolInstance string) *Toolrecord {
now := time.Now().UTC()
reportFileName := filepath.Join(workspace,
"toolruns",
"toolrun_"+toolName+"_"+
now.Format("20210731")+
strings.ReplaceAll(now.Format("15:04:05"), ":", "")+
now.Format("20060102150405")+
".json")
tr.reportFileName = reportFileName
@ -106,6 +106,12 @@ func (tr *Toolrecord) Persist() error {
if tr.ToolInstance == "" {
return errors.New("TR_PERSIST: empty instanceName")
}
// create workspace/toolrecord
dirPath := filepath.Join(tr.workspace, "toolruns")
err := os.MkdirAll(dirPath, os.ModePerm)
if err != nil {
return fmt.Errorf("TR_PERSIST: %v", err)
}
// convenience aggregation
displayName := ""
displayURL := ""
@ -127,7 +133,7 @@ func (tr *Toolrecord) Persist() error {
tr.DisplayURL = displayURL
file, _ := json.Marshal(tr)
err := ioutil.WriteFile(tr.GetFileName(), file, 0644)
err = ioutil.WriteFile(tr.GetFileName(), file, 0644)
if err != nil {
return fmt.Errorf("TR_PERSIST: %v", err)
}