1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-02-21 19:48:53 +02:00

Fortify: Using mvn to auto-resolve classpath needs additional params (#1607)

* also reduce code duplication in token fetching
* concatenate classpaths from multi-maven projects

Co-authored-by: Daniel Kurzynski <daniel.kurzynski@sap.com>
This commit is contained in:
Stephan Aßmus 2020-05-29 15:42:35 +02:00 committed by GitHub
parent ae6853ee4d
commit a24a7aad23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 176 additions and 110 deletions

View File

@ -5,9 +5,11 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/bmatcuk/doublestar"
"io/ioutil"
"math"
"os"
"path/filepath"
"regexp"
"strings"
"time"
@ -33,7 +35,7 @@ type pullRequestService interface {
}
const checkString = "<---CHECK FORTIFY---"
const classpathFileName = "cp.txt"
const classpathFileName = "fortify-execute-scan-cp.txt"
func fortifyExecuteScan(config fortifyExecuteScanOptions, telemetryData *telemetry.CustomData, influx *fortifyExecuteScanInflux) {
auditStatus := map[string]string{}
@ -474,18 +476,46 @@ func autoresolvePipClasspath(executable string, parameters []string, file string
return readClasspathFile(file)
}
func autoresolveMavenClasspath(pomFilePath, file string, command execRunner) string {
func autoresolveMavenClasspath(config fortifyExecuteScanOptions, file string, command execRunner) string {
if filepath.IsAbs(file) {
log.Entry().Warnf("Passing an absolute path for -Dmdep.outputFile results in the classpath only for the last module in multi-module maven projects.")
}
executeOptions := maven.ExecuteOptions{
PomPath: pomFilePath,
Goals: []string{"dependency:build-classpath"},
Defines: []string{fmt.Sprintf("-Dmdep.outputFile=%v", file), "-DincludeScope=compile"},
ReturnStdout: false,
PomPath: config.BuildDescriptorFile,
ProjectSettingsFile: config.ProjectSettingsFile,
GlobalSettingsFile: config.GlobalSettingsFile,
M2Path: config.M2Path,
Goals: []string{"dependency:build-classpath"},
Defines: []string{fmt.Sprintf("-Dmdep.outputFile=%v", file), "-DincludeScope=compile"},
ReturnStdout: false,
}
if len(strings.TrimSpace(config.MvnCustomArgs)) > 0 {
executeOptions.Flags = tokenize(config.MvnCustomArgs)
}
_, err := maven.Execute(&executeOptions, command)
if err != nil {
log.Entry().WithError(err).Warn("failed to determine classpath using Maven")
}
return readClasspathFile(file)
return readAllClasspathFiles(file)
}
// readAllClasspathFiles tests whether the passed file is an absolute path. If not, it will glob for
// all files under the current directory with the given file name and concatenate their contents.
// Otherwise it will return the contents pointed to by the absolute path.
func readAllClasspathFiles(file string) string {
var paths []string
if filepath.IsAbs(file) {
paths = []string{file}
} else {
paths, _ = doublestar.Glob(filepath.Join("**", file))
log.Entry().Debugf("Concatenating the class paths from %v", paths)
}
var contents string
const separator = ":"
for _, path := range paths {
contents += separator + readClasspathFile(path)
}
return removeDuplicates(contents, separator)
}
func readClasspathFile(file string) string {
@ -493,7 +523,35 @@ func readClasspathFile(file string) string {
if err != nil {
log.Entry().WithError(err).Warnf("failed to read classpath from file '%v'", file)
}
return strings.TrimSpace(string(data))
result := strings.TrimSpace(string(data))
if len(result) == 0 {
log.Entry().Warnf("classpath from file '%v' was empty", file)
}
return result
}
func removeDuplicates(contents, separator string) string {
if separator == "" || contents == "" {
return contents
}
entries := strings.Split(contents, separator)
entrySet := map[string]struct{}{}
contents = ""
for _, entry := range entries {
if entry == "" {
continue
}
_, contained := entrySet[entry]
if !contained {
entrySet[entry] = struct{}{}
contents += entry + separator
}
}
if contents != "" {
// Remove trailing "separator"
contents = contents[:len(contents)-len(separator)]
}
return contents
}
func triggerFortifyScan(config fortifyExecuteScanOptions, command execRunner, buildID, buildLabel, buildProject string) {
@ -507,7 +565,7 @@ func triggerFortifyScan(config fortifyExecuteScanOptions, command execRunner, bu
classpath := ""
if config.BuildTool == "maven" {
if config.AutodetectClasspath {
classpath = autoresolveMavenClasspath(config.BuildDescriptorFile, classpathFileName, command)
classpath = autoresolveMavenClasspath(config, classpathFileName, command)
}
config.Translate, err = populateMavenTranslate(&config, classpath)
if err != nil {
@ -687,6 +745,8 @@ func appendToOptions(config *fortifyExecuteScanOptions, options []string, t map[
options = append(options, "-cp", t["autoClasspath"])
} else if len(t["classpath"]) > 0 {
options = append(options, "-cp", t["classpath"])
} else {
log.Entry().Debugf("no field 'autoClasspath' or 'classpath' in map or both empty")
}
if len(t["extdirs"]) > 0 {
options = append(options, "-extdirs", t["extdirs"])

View File

@ -65,6 +65,9 @@ type fortifyExecuteScanOptions struct {
PullRequestName string `json:"pullRequestName,omitempty"`
PullRequestMessageRegex string `json:"pullRequestMessageRegex,omitempty"`
BuildTool string `json:"buildTool,omitempty"`
ProjectSettingsFile string `json:"projectSettingsFile,omitempty"`
GlobalSettingsFile string `json:"globalSettingsFile,omitempty"`
M2Path string `json:"m2Path,omitempty"`
}
type fortifyExecuteScanInflux struct {
@ -236,6 +239,9 @@ func addFortifyExecuteScanFlags(cmd *cobra.Command, stepConfig *fortifyExecuteSc
cmd.Flags().StringVar(&stepConfig.PullRequestName, "pullRequestName", os.Getenv("PIPER_pullRequestName"), "The name of the pull request branch which will trigger creation of a new version in Fortify SSC based on the master branch version")
cmd.Flags().StringVar(&stepConfig.PullRequestMessageRegex, "pullRequestMessageRegex", `.*Merge pull request #(\\d+) from.*`, "Regex used to identify the PR-XXX reference within the merge commit message")
cmd.Flags().StringVar(&stepConfig.BuildTool, "buildTool", `maven`, "Scan type used for the step which can be `'maven'`, `'pip'`")
cmd.Flags().StringVar(&stepConfig.ProjectSettingsFile, "projectSettingsFile", os.Getenv("PIPER_projectSettingsFile"), "Path to the mvn settings file that should be used as project settings file.")
cmd.Flags().StringVar(&stepConfig.GlobalSettingsFile, "globalSettingsFile", os.Getenv("PIPER_globalSettingsFile"), "Path to the mvn settings file that should be used as global settings file.")
cmd.Flags().StringVar(&stepConfig.M2Path, "m2Path", os.Getenv("PIPER_m2Path"), "Path to the location of the local repository that should be used.")
cmd.MarkFlagRequired("authToken")
}
@ -642,6 +648,30 @@ func fortifyExecuteScanMetadata() config.StepData {
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "projectSettingsFile",
ResourceRef: []config.ResourceReference{},
Scope: []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{{Name: "maven/projectSettingsFile"}},
},
{
Name: "globalSettingsFile",
ResourceRef: []config.ResourceReference{},
Scope: []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{{Name: "maven/globalSettingsFile"}},
},
{
Name: "m2Path",
ResourceRef: []config.ResourceReference{},
Scope: []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{{Name: "maven/m2Path"}},
},
},
},
},

View File

@ -376,7 +376,7 @@ func TestTriggerFortifyScan(t *testing.T) {
assert.Equal(t, 3, runner.numExecutions)
assert.Equal(t, "mvn", runner.executions[0].executable)
assert.Equal(t, []string{"--file", "./pom.xml", "-Dmdep.outputFile=cp.txt", "-DincludeScope=compile", "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn", "--batch-mode", "dependency:build-classpath"}, runner.executions[0].parameters)
assert.Equal(t, []string{"--file", "./pom.xml", "-Dmdep.outputFile=fortify-execute-scan-cp.txt", "-DincludeScope=compile", "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn", "--batch-mode", "dependency:build-classpath"}, runner.executions[0].parameters)
assert.Equal(t, "sourceanalyzer", runner.executions[1].executable)
assert.Equal(t, []string{"-verbose", "-64", "-b", "test", "-Xmx4G", "-Xms2G", "-cp", "some.jar;someother.jar", "**/*.xml", "**/*.html", "**/*.jsp", "**/*.js", "src/main/resources/**/*", "src/main/java/**/*"}, runner.executions[1].parameters)
@ -604,7 +604,7 @@ func TestAutoresolveClasspath(t *testing.T) {
defer os.RemoveAll(dir)
file := filepath.Join(dir, "cp.txt")
result := autoresolveMavenClasspath("pom.xml", file, &execRunner)
result := autoresolveMavenClasspath(fortifyExecuteScanOptions{BuildDescriptorFile: "pom.xml"}, file, &execRunner)
assert.Equal(t, "mvn", execRunner.executions[0].executable, "Expected different executable")
assert.Equal(t, []string{"--file", "pom.xml", fmt.Sprintf("-Dmdep.outputFile=%v", file), "-DincludeScope=compile", "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn", "--batch-mode", "dependency:build-classpath"}, execRunner.executions[0].parameters, "Expected different parameters")
assert.Equal(t, "some.jar;someother.jar", result, "Expected different result")
@ -664,3 +664,23 @@ func TestPopulatePipTranslate(t *testing.T) {
assert.Equal(t, `[{"pythonPath":""}]`, translate, "Expected different parameters")
})
}
func TestRemoveDuplicates(t *testing.T) {
testData := []struct {
name string
input string
expected string
separator string
}{
{"empty", "", "", "x"},
{"no duplicates", ":a::b::", "a:b", ":"},
{"duplicates", "::a:b:a:b::a", "a:b", ":"},
{"long separator", "..a.b....ab..a.b", "a.b..ab", ".."},
{"no separator", "abc", "abc", ""},
}
for _, data := range testData {
t.Run(data.name, func(t *testing.T) {
assert.Equal(t, data.expected, removeDuplicates(data.input, data.separator))
})
}
}

View File

@ -588,9 +588,9 @@ func (sys *SystemInstance) GenerateQGateReport(projectID, projectVersionID, repo
paramType := "SINGLE_PROJECT"
paramName := "Q-gate-report"
reportType := "PORTFOLIO"
inputReportParameters := []*models.InputReportParameter{&models.InputReportParameter{Name: &paramName, Identifier: &paramIdentifier, ParamValue: projectVersionID, Type: &paramType}}
reportProjectVersions := []*models.ReportProjectVersion{&models.ReportProjectVersion{ID: projectVersionID, Name: projectVersionName}}
reportProjects := []*models.ReportProject{&models.ReportProject{ID: projectID, Name: projectName, Versions: reportProjectVersions}}
inputReportParameters := []*models.InputReportParameter{{Name: &paramName, Identifier: &paramIdentifier, ParamValue: projectVersionID, Type: &paramType}}
reportProjectVersions := []*models.ReportProjectVersion{{ID: projectVersionID, Name: projectVersionName}}
reportProjects := []*models.ReportProject{{ID: projectID, Name: projectName, Versions: reportProjectVersions}}
report := models.SavedReport{Name: fmt.Sprintf("FortifyReport: %v:%v", projectName, projectVersionName), Type: &reportType, ReportDefinitionID: &reportTemplateID, Format: &reportFormat, Projects: reportProjects, InputReportParameters: inputReportParameters}
params := &saved_report_controller.CreateSavedReportParams{Resource: &report}
params.WithTimeout(sys.timeout)
@ -621,6 +621,7 @@ func (sys *SystemInstance) invalidateFileTokens() error {
}
func (sys *SystemInstance) getFileToken(tokenType string) (*models.FileToken, error) {
log.Entry().Debugf("fetching file token of type %v", tokenType)
token := models.FileToken{FileTokenType: &tokenType}
params := &file_token_controller.CreateFileTokenParams{Resource: &token}
params.WithTimeout(sys.timeout)
@ -631,21 +632,6 @@ func (sys *SystemInstance) getFileToken(tokenType string) (*models.FileToken, er
return result.GetPayload().Data, nil
}
func (sys *SystemInstance) getFileUploadToken() (*models.FileToken, error) {
log.Entry().Debug("fetching upload token")
return sys.getFileToken("UPLOAD")
}
func (sys *SystemInstance) getFileDownloadToken() (*models.FileToken, error) {
log.Entry().Debug("fetching download token")
return sys.getFileToken("DOWNLOAD")
}
func (sys *SystemInstance) getReportFileToken() (*models.FileToken, error) {
log.Entry().Debug("fetching report download token")
return sys.getFileToken("REPORT_FILE")
}
// UploadResultFile uploads a fpr file to the fortify backend
func (sys *SystemInstance) UploadResultFile(endpoint, file string, projectVersionID int64) error {
fileHandle, err := os.Open(file)
@ -658,7 +644,7 @@ func (sys *SystemInstance) UploadResultFile(endpoint, file string, projectVersio
}
func (sys *SystemInstance) uploadResultFileContent(endpoint, file string, fileContent io.Reader, projectVersionID int64) error {
token, err := sys.getFileUploadToken()
token, err := sys.getFileToken("UPLOAD")
if err != nil {
return err
}
@ -684,7 +670,13 @@ func (sys *SystemInstance) uploadResultFileContent(endpoint, file string, fileCo
}
// DownloadFile downloads a file from Fortify backend
func (sys *SystemInstance) downloadFile(endpoint, method, acceptType, downloadToken string, projectVersionID int64) ([]byte, error) {
func (sys *SystemInstance) downloadFile(endpoint, method, acceptType, tokenType string, projectVersionID int64) ([]byte, error) {
token, err := sys.getFileToken(tokenType)
if err != nil {
return nil, errors.Wrap(err, "Error fetching file token")
}
defer sys.invalidateFileTokens()
header := http.Header{}
header.Add("Cache-Control", "no-cache, no-store, must-revalidate")
header.Add("Pragma", "no-cache")
@ -692,10 +684,9 @@ func (sys *SystemInstance) downloadFile(endpoint, method, acceptType, downloadTo
header.Add("Content-Type", "application/form-data")
body := url.Values{
"id": {fmt.Sprintf("%v", projectVersionID)},
"mat": {downloadToken},
"mat": {token.Token},
}
var response *http.Response
var err error
if method == http.MethodGet {
response, err = sys.httpClient.SendRequest(method, fmt.Sprintf("%v%v?%v", sys.serverURL, endpoint, body.Encode()), nil, header, nil)
} else {
@ -714,12 +705,7 @@ func (sys *SystemInstance) downloadFile(endpoint, method, acceptType, downloadTo
// DownloadReportFile downloads a report file from Fortify backend
func (sys *SystemInstance) DownloadReportFile(endpoint string, projectVersionID int64) ([]byte, error) {
token, err := sys.getReportFileToken()
if err != nil {
return nil, errors.Wrap(err, "Error fetching report download token")
}
defer sys.invalidateFileTokens()
data, err := sys.downloadFile(endpoint, http.MethodGet, "application/octet-stream", token.Token, projectVersionID)
data, err := sys.downloadFile(endpoint, http.MethodGet, "application/octet-stream", "REPORT_FILE", projectVersionID)
if err != nil {
return nil, errors.Wrap(err, "Error downloading report file")
}
@ -728,12 +714,7 @@ func (sys *SystemInstance) DownloadReportFile(endpoint string, projectVersionID
// DownloadResultFile downloads a result file from Fortify backend
func (sys *SystemInstance) DownloadResultFile(endpoint string, projectVersionID int64) ([]byte, error) {
token, err := sys.getFileDownloadToken()
if err != nil {
return nil, errors.Wrap(err, "Error fetching result file download token")
}
defer sys.invalidateFileTokens()
data, err := sys.downloadFile(endpoint, http.MethodGet, "application/zip", token.Token, projectVersionID)
data, err := sys.downloadFile(endpoint, http.MethodGet, "application/zip", "DOWNLOAD", projectVersionID)
if err != nil {
return nil, errors.Wrap(err, "Error downloading result file")
}

View File

@ -328,7 +328,7 @@ func TestSetProjectVersionAttributesByProjectVersionID(t *testing.T) {
t.Run("test success", func(t *testing.T) {
value := "abcd"
defID := int64(18)
attributes := []*models.Attribute{&models.Attribute{ID: 4712, Value: &value, AttributeDefinitionID: &defID}}
attributes := []*models.Attribute{{ID: 4712, Value: &value, AttributeDefinitionID: &defID}}
result, err := sys.SetProjectVersionAttributesByProjectVersionID(4711, attributes)
assert.NoError(t, err, "SetProjectVersionAttributesByProjectVersionID call not successful")
assert.Equal(t, 1, len(result), "Expected to get slice with different amount of values")
@ -670,7 +670,7 @@ func TestReduceIssueFilterSelectorSet(t *testing.T) {
name1 := "Special"
name2 := "Other"
guid := "FOLDER"
options := []*models.SelectorOption{&models.SelectorOption{GUID: "1234567", DisplayName: "Test"}, &models.SelectorOption{GUID: "1234568", DisplayName: "Test2"}}
options := []*models.SelectorOption{{GUID: "1234567", DisplayName: "Test"}, {GUID: "1234568", DisplayName: "Test2"}}
filterSet := models.IssueFilterSelectorSet{FilterBySet: []*models.IssueFilterSelector{}, GroupBySet: []*models.IssueSelector{}}
filterSet.FilterBySet = append(filterSet.FilterBySet, &models.IssueFilterSelector{DisplayName: name1, SelectorOptions: options})
filterSet.FilterBySet = append(filterSet.FilterBySet, &models.IssueFilterSelector{DisplayName: name2})
@ -860,12 +860,12 @@ func TestGetReportDetails(t *testing.T) {
})
}
func TestGetFileUploadToken(t *testing.T) {
func TestGetFileToken(t *testing.T) {
// Start a local HTTP server
bodyContent := ""
reference := `{"fileTokenType":"UPLOAD"}
reference := `{"fileTokenType":"TOKEN_TYPE"}
`
response := `{"data": {"fileTokenType": "UPLOAD","token": "ZjE1OTdjZjEtMjAzNS00NTFmLThiOWItNzBkYzI0MWEzZGNj"},"responseCode": 201}`
response := `{"data": {"fileTokenType": "TOKEN_TYPE","token": "ZjE1OTdjZjEtMjAzNS00NTFmLThiOWItNzBkYzI0MWEzZGNj"},"responseCode": 201}`
sys, server := spinUpServer(func(rw http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/fileTokens" {
header := rw.Header()
@ -881,66 +881,10 @@ func TestGetFileUploadToken(t *testing.T) {
defer server.Close()
t.Run("test success", func(t *testing.T) {
result, err := sys.getFileUploadToken()
assert.NoError(t, err, "getFileUploadToken call not successful")
assert.Equal(t, "ZjE1OTdjZjEtMjAzNS00NTFmLThiOWItNzBkYzI0MWEzZGNj", result.Token, "Different result content expected")
assert.Equal(t, reference, bodyContent, "Different request content expected")
})
}
func TestGetFileDownloadToken(t *testing.T) {
// Start a local HTTP server
bodyContent := ""
reference := `{"fileTokenType":"DOWNLOAD"}
`
response := `{"data": {"fileTokenType": "DOWNLOAD","token": "ZjE1OTdjZjEtMjAzNS00NTFmLThiOWItNzBkYzI0MWEzZGNj"},"responseCode": 201}`
sys, server := spinUpServer(func(rw http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/fileTokens" {
header := rw.Header()
header.Add("Content-type", "application/json")
bodyBytes, _ := ioutil.ReadAll(req.Body)
bodyContent = string(bodyBytes)
rw.WriteHeader(201)
rw.Write([]byte(response))
return
}
})
// Close the server when test finishes
defer server.Close()
t.Run("test success", func(t *testing.T) {
result, err := sys.getFileDownloadToken()
assert.NoError(t, err, "getFileDownloadToken call not successful")
assert.Equal(t, "ZjE1OTdjZjEtMjAzNS00NTFmLThiOWItNzBkYzI0MWEzZGNj", result.Token, "Different result content expected")
assert.Equal(t, reference, bodyContent, "Different request content expected")
})
}
func TestGetReportFileToken(t *testing.T) {
// Start a local HTTP server
bodyContent := ""
reference := `{"fileTokenType":"REPORT_FILE"}
`
response := `{"data": {"fileTokenType": "REPORT_FILE","token": "ZjE1OTdjZjEtMjAzNS00NTFmLThiOWItNzBkYzI0MWEzZGNj"},"responseCode": 201}`
sys, server := spinUpServer(func(rw http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/fileTokens" {
header := rw.Header()
header.Add("Content-type", "application/json")
bodyBytes, _ := ioutil.ReadAll(req.Body)
bodyContent = string(bodyBytes)
rw.WriteHeader(201)
rw.Write([]byte(response))
return
}
})
// Close the server when test finishes
defer server.Close()
t.Run("test success", func(t *testing.T) {
result, err := sys.getReportFileToken()
assert.NoError(t, err, "getReportFileToken call not successful")
assert.Equal(t, "ZjE1OTdjZjEtMjAzNS00NTFmLThiOWItNzBkYzI0MWEzZGNj", result.Token, "Different result content expected")
assert.Equal(t, reference, bodyContent, "Different request content expected")
result, err := sys.getFileToken("TOKEN_TYPE")
assert.NoError(t, err)
assert.Equal(t, "ZjE1OTdjZjEtMjAzNS00NTFmLThiOWItNzBkYzI0MWEzZGNj", result.Token)
assert.Equal(t, reference, bodyContent)
})
}

View File

@ -462,6 +462,37 @@ spec:
- STAGES
- STEPS
default: maven
# Global maven settings, should be added to all maven steps
- name: projectSettingsFile
type: string
description: Path to the mvn settings file that should be used as project settings file.
scope:
- GENERAL
- STEPS
- STAGES
- PARAMETERS
aliases:
- name: maven/projectSettingsFile
- name: globalSettingsFile
type: string
description: Path to the mvn settings file that should be used as global settings file.
scope:
- GENERAL
- STEPS
- STAGES
- PARAMETERS
aliases:
- name: maven/globalSettingsFile
- name: m2Path
type: string
description: Path to the location of the local repository that should be used.
scope:
- GENERAL
- STEPS
- STAGES
- PARAMETERS
aliases:
- name: maven/m2Path
containers:
- image: ppiper/fortify
workingDir: /home/piper