2020-05-25 19:48:59 +02:00
package cmd
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"math"
"os"
2020-05-29 15:42:35 +02:00
"path/filepath"
2020-05-25 19:48:59 +02:00
"regexp"
2020-06-02 13:47:07 +02:00
"runtime"
2021-06-23 15:05:00 +02:00
"strconv"
2020-05-25 19:48:59 +02:00
"strings"
"time"
2021-02-26 14:43:03 +02:00
piperhttp "github.com/SAP/jenkins-library/pkg/http"
2020-07-27 12:01:59 +02:00
"github.com/bmatcuk/doublestar"
2020-09-14 12:05:12 +02:00
"github.com/google/go-github/v32/github"
2020-05-25 19:48:59 +02:00
"github.com/google/uuid"
"github.com/piper-validation/fortify-client-go/models"
"github.com/SAP/jenkins-library/pkg/command"
"github.com/SAP/jenkins-library/pkg/fortify"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/maven"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/SAP/jenkins-library/pkg/telemetry"
2021-06-23 15:05:00 +02:00
"github.com/SAP/jenkins-library/pkg/toolrecord"
2020-05-25 19:48:59 +02:00
"github.com/SAP/jenkins-library/pkg/versioning"
piperGithub "github.com/SAP/jenkins-library/pkg/github"
2020-11-11 14:04:45 +02:00
"github.com/pkg/errors"
2020-05-25 19:48:59 +02:00
)
type pullRequestService interface {
ListPullRequestsWithCommit ( ctx context . Context , owner , repo , sha string , opts * github . PullRequestListOptions ) ( [ ] * github . PullRequest , * github . Response , error )
}
2020-11-10 18:14:55 +02:00
type fortifyUtils interface {
maven . Utils
2020-06-16 11:42:51 +02:00
SetDir ( d string )
2020-11-10 18:14:55 +02:00
GetArtifact ( buildTool , buildDescriptorFile string ,
options * versioning . Options ) ( versioning . Artifact , error )
}
type fortifyUtilsBundle struct {
* command . Command
* piperutils . Files
* piperhttp . Client
}
func ( bundle * fortifyUtilsBundle ) GetArtifact ( buildTool , buildDescriptorFile string ,
options * versioning . Options ) ( versioning . Artifact , error ) {
return versioning . GetArtifact ( buildTool , buildDescriptorFile , options , bundle )
}
2021-02-26 14:43:03 +02:00
func newFortifyUtilsBundle ( ) fortifyUtils {
2020-11-10 18:14:55 +02:00
utils := fortifyUtilsBundle {
Command : & command . Command { } ,
Files : & piperutils . Files { } ,
Client : & piperhttp . Client { } ,
}
utils . Stdout ( log . Writer ( ) )
utils . Stderr ( log . Writer ( ) )
return & utils
2020-06-16 11:42:51 +02:00
}
2020-05-25 19:48:59 +02:00
const checkString = "<---CHECK FORTIFY---"
2020-05-29 15:42:35 +02:00
const classpathFileName = "fortify-execute-scan-cp.txt"
2020-05-25 19:48:59 +02:00
func fortifyExecuteScan ( config fortifyExecuteScanOptions , telemetryData * telemetry . CustomData , influx * fortifyExecuteScanInflux ) {
auditStatus := map [ string ] string { }
2020-10-27 12:11:53 +02:00
sys := fortify . NewSystemInstance ( config . ServerURL , config . APIEndpoint , config . AuthToken , time . Minute * 15 )
2021-02-26 14:43:03 +02:00
utils := newFortifyUtilsBundle ( )
2020-09-18 08:19:34 +02:00
2021-03-18 11:32:03 +02:00
influx . step_data . fields . fortify = false
2020-11-10 18:14:55 +02:00
reports , err := runFortifyScan ( config , sys , utils , telemetryData , influx , auditStatus )
2020-09-18 08:19:34 +02:00
piperutils . PersistReportsAndLinks ( "fortifyExecuteScan" , config . ModulePath , reports , nil )
if err != nil {
log . Entry ( ) . WithError ( err ) . Fatal ( "Fortify scan and check failed" )
2020-05-25 19:48:59 +02:00
}
2021-03-18 11:32:03 +02:00
influx . step_data . fields . fortify = true
2020-11-11 14:04:45 +02:00
// make sure that no specific error category is set in success case
log . SetErrorCategory ( log . ErrorUndefined )
2020-05-25 19:48:59 +02:00
}
2020-11-10 18:14:55 +02:00
func determineArtifact ( config fortifyExecuteScanOptions , utils fortifyUtils ) ( versioning . Artifact , error ) {
2020-08-07 10:31:15 +02:00
versioningOptions := versioning . Options {
M2Path : config . M2Path ,
GlobalSettingsFile : config . GlobalSettingsFile ,
ProjectSettingsFile : config . ProjectSettingsFile ,
}
2020-09-18 08:19:34 +02:00
2020-11-10 18:14:55 +02:00
artifact , err := utils . GetArtifact ( config . BuildTool , config . BuildDescriptorFile , & versioningOptions )
2020-05-25 19:48:59 +02:00
if err != nil {
2020-09-18 08:19:34 +02:00
return nil , fmt . Errorf ( "Unable to get artifact from descriptor %v: %w" , config . BuildDescriptorFile , err )
2020-05-25 19:48:59 +02:00
}
2020-09-18 08:19:34 +02:00
return artifact , nil
}
2020-11-10 18:14:55 +02:00
func runFortifyScan ( config fortifyExecuteScanOptions , sys fortify . System , utils fortifyUtils , telemetryData * telemetry . CustomData , influx * fortifyExecuteScanInflux , auditStatus map [ string ] string ) ( [ ] piperutils . Path , error ) {
2020-09-18 08:19:34 +02:00
var reports [ ] piperutils . Path
log . Entry ( ) . Debugf ( "Running Fortify scan against SSC at %v" , config . ServerURL )
2020-11-10 18:14:55 +02:00
2020-11-16 11:29:21 +02:00
if config . BuildTool == "maven" && config . InstallArtifacts {
err := maven . InstallMavenArtifacts ( & maven . EvaluateOptions {
M2Path : config . M2Path ,
ProjectSettingsFile : config . ProjectSettingsFile ,
GlobalSettingsFile : config . GlobalSettingsFile ,
PomPath : config . BuildDescriptorFile ,
} , utils )
if err != nil {
return reports , fmt . Errorf ( "Unable to install artifacts: %w" , err )
}
}
2020-11-10 18:14:55 +02:00
artifact , err := determineArtifact ( config , utils )
if err != nil {
log . Entry ( ) . WithError ( err ) . Fatal ( )
}
2020-08-07 10:31:15 +02:00
coordinates , err := artifact . GetCoordinates ( )
2020-05-25 19:48:59 +02:00
if err != nil {
2020-11-11 14:04:45 +02:00
log . SetErrorCategory ( log . ErrorConfiguration )
2020-09-18 08:19:34 +02:00
return reports , fmt . Errorf ( "unable to get project coordinates from descriptor %v: %w" , config . BuildDescriptorFile , err )
2020-05-25 19:48:59 +02:00
}
2021-02-26 14:43:03 +02:00
log . Entry ( ) . Debugf ( "loaded project coordinates %v from descriptor" , coordinates )
if len ( config . Version ) > 0 {
log . Entry ( ) . Infof ( "Resolving product version from default provided '%s' with versioning '%s'" , config . Version , config . VersioningModel )
coordinates . Version = config . Version
}
fortifyProjectName , fortifyProjectVersion := versioning . DetermineProjectCoordinatesWithCustomVersion ( config . ProjectName , config . VersioningModel , config . CustomScanVersion , coordinates )
2020-05-25 19:48:59 +02:00
project , err := sys . GetProjectByName ( fortifyProjectName , config . AutoCreate , fortifyProjectVersion )
if err != nil {
2021-02-26 14:43:03 +02:00
classifyErrorOnLookup ( err )
2020-09-18 08:19:34 +02:00
return reports , fmt . Errorf ( "Failed to load project %v: %w" , fortifyProjectName , err )
2020-05-25 19:48:59 +02:00
}
projectVersion , err := sys . GetProjectVersionDetailsByProjectIDAndVersionName ( project . ID , fortifyProjectVersion , config . AutoCreate , fortifyProjectName )
if err != nil {
2021-02-26 14:43:03 +02:00
classifyErrorOnLookup ( err )
2020-09-18 08:19:34 +02:00
return reports , fmt . Errorf ( "Failed to load project version %v: %w" , fortifyProjectVersion , err )
2020-05-25 19:48:59 +02:00
}
if len ( config . PullRequestName ) > 0 {
fortifyProjectVersion = config . PullRequestName
2021-03-04 10:34:05 +02:00
projectVersion , err = sys . LookupOrCreateProjectVersionDetailsForPullRequest ( project . ID , projectVersion , fortifyProjectVersion )
2020-05-25 19:48:59 +02:00
if err != nil {
2021-02-26 14:43:03 +02:00
classifyErrorOnLookup ( err )
2020-09-18 08:19:34 +02:00
return reports , fmt . Errorf ( "Failed to lookup / create project version for pull request %v: %w" , fortifyProjectVersion , err )
2020-05-25 19:48:59 +02:00
}
log . Entry ( ) . Debugf ( "Looked up / created project version with ID %v for PR %v" , projectVersion . ID , fortifyProjectVersion )
} else {
2022-01-21 11:52:17 +02:00
prID , prAuthor := determinePullRequestMerge ( config )
2022-01-24 12:59:33 +02:00
if prID != "0" {
2020-05-25 19:48:59 +02:00
log . Entry ( ) . Debugf ( "Determined PR ID '%v' for merge check" , prID )
2022-01-21 11:52:17 +02:00
if len ( prAuthor ) > 0 && ! piperutils . ContainsString ( config . Assignees , prAuthor ) {
log . Entry ( ) . Debugf ( "Determined PR Author '%v' for result assignment" , prAuthor )
config . Assignees = append ( config . Assignees , prAuthor )
}
2020-05-25 19:48:59 +02:00
pullRequestProjectName := fmt . Sprintf ( "PR-%v" , prID )
err = sys . MergeProjectVersionStateOfPRIntoMaster ( config . FprDownloadEndpoint , config . FprUploadEndpoint , project . ID , projectVersion . ID , pullRequestProjectName )
if err != nil {
2020-09-18 08:19:34 +02:00
return reports , fmt . Errorf ( "Failed to merge project version state for pull request %v into project version %v of project %v: %w" , pullRequestProjectName , fortifyProjectVersion , project . ID , err )
2020-05-25 19:48:59 +02:00
}
}
}
2020-09-18 08:19:34 +02:00
filterSet , err := sys . GetFilterSetOfProjectVersionByTitle ( projectVersion . ID , config . FilterSetTitle )
if filterSet == nil || err != nil {
return reports , fmt . Errorf ( "Failed to load filter set with title %v" , config . FilterSetTitle )
}
2021-06-23 15:05:00 +02:00
// create toolrecord file
// tbd - how to handle verifyOnly
2021-07-23 08:48:48 +02:00
toolRecordFileName , err := createToolRecordFortify ( "./" , config , project . ID , fortifyProjectName , projectVersion . ID , fortifyProjectVersion )
2021-06-23 15:05:00 +02:00
if err != nil {
// do not fail until the framework is well established
log . Entry ( ) . Warning ( "TR_FORTIFY: Failed to create toolrecord file ..." , err )
} else {
reports = append ( reports , piperutils . Path { Target : toolRecordFileName } )
}
2020-09-18 08:19:34 +02:00
if config . VerifyOnly {
log . Entry ( ) . Infof ( "Starting audit status check on project %v with version %v and project version ID %v" , fortifyProjectName , fortifyProjectVersion , projectVersion . ID )
2021-06-15 14:53:42 +02:00
err , paths := verifyFFProjectCompliance ( config , sys , project , projectVersion , filterSet , influx , auditStatus )
reports = append ( reports , paths ... )
return reports , err
2020-09-18 08:19:34 +02:00
}
log . Entry ( ) . Infof ( "Scanning and uploading to project %v with version %v and projectVersionId %v" , fortifyProjectName , fortifyProjectVersion , projectVersion . ID )
2020-05-25 19:48:59 +02:00
buildLabel := fmt . Sprintf ( "%v/repos/%v/%v/commits/%v" , config . GithubAPIURL , config . Owner , config . Repository , config . CommitID )
// Create sourceanalyzer command based on configuration
buildID := uuid . New ( ) . String ( )
2020-11-10 18:14:55 +02:00
utils . SetDir ( config . ModulePath )
2020-05-25 19:48:59 +02:00
os . MkdirAll ( fmt . Sprintf ( "%v/%v" , config . ModulePath , "target" ) , os . ModePerm )
if config . UpdateRulePack {
2020-11-10 18:14:55 +02:00
err := utils . RunExecutable ( "fortifyupdate" , "-acceptKey" , "-acceptSSLCertificate" , "-url" , config . ServerURL )
2020-05-25 19:48:59 +02:00
if err != nil {
2020-11-11 14:04:45 +02:00
return reports , fmt . Errorf ( "failed to update rule pack, serverUrl: %v" , config . ServerURL )
2020-05-25 19:48:59 +02:00
}
2020-11-10 18:14:55 +02:00
err = utils . RunExecutable ( "fortifyupdate" , "-acceptKey" , "-acceptSSLCertificate" , "-showInstalledRules" )
2020-05-25 19:48:59 +02:00
if err != nil {
2020-11-11 14:04:45 +02:00
return reports , fmt . Errorf ( "failed to fetch details of installed rule pack, serverUrl: %v" , config . ServerURL )
2020-05-25 19:48:59 +02:00
}
}
2021-06-15 14:53:42 +02:00
err = triggerFortifyScan ( config , utils , buildID , buildLabel , fortifyProjectName )
2020-05-25 19:48:59 +02:00
reports = append ( reports , piperutils . Path { Target : fmt . Sprintf ( "%vtarget/fortify-scan.*" , config . ModulePath ) } )
reports = append ( reports , piperutils . Path { Target : fmt . Sprintf ( "%vtarget/*.fpr" , config . ModulePath ) } )
2021-06-15 14:53:42 +02:00
if err != nil {
return reports , errors . Wrapf ( err , "failed to scan project" )
}
2020-05-25 19:48:59 +02:00
var message string
if config . UploadResults {
log . Entry ( ) . Debug ( "Uploading results" )
resultFilePath := fmt . Sprintf ( "%vtarget/result.fpr" , config . ModulePath )
err = sys . UploadResultFile ( config . FprUploadEndpoint , resultFilePath , projectVersion . ID )
message = fmt . Sprintf ( "Failed to upload result file %v to Fortify SSC at %v" , resultFilePath , config . ServerURL )
} else {
log . Entry ( ) . Debug ( "Generating XML report" )
xmlReportName := "fortify_result.xml"
2020-11-10 18:14:55 +02:00
err = utils . RunExecutable ( "ReportGenerator" , "-format" , "xml" , "-f" , xmlReportName , "-source" , fmt . Sprintf ( "%vtarget/result.fpr" , config . ModulePath ) )
2020-05-25 19:48:59 +02:00
message = fmt . Sprintf ( "Failed to generate XML report %v" , xmlReportName )
if err != nil {
reports = append ( reports , piperutils . Path { Target : fmt . Sprintf ( "%vfortify_result.xml" , config . ModulePath ) } )
}
}
if err != nil {
2020-09-18 08:19:34 +02:00
return reports , fmt . Errorf ( message + ": %w" , err )
2020-05-25 19:48:59 +02:00
}
2020-09-18 08:19:34 +02:00
log . Entry ( ) . Infof ( "Starting audit status check on project %v with version %v and project version ID %v" , fortifyProjectName , fortifyProjectVersion , projectVersion . ID )
2020-05-25 19:48:59 +02:00
// Ensure latest FPR is processed
2020-08-11 15:29:00 +02:00
err = verifyScanResultsFinishedUploading ( config , sys , projectVersion . ID , buildLabel , filterSet ,
10 * time . Second , time . Duration ( config . PollingMinutes ) * time . Minute )
2020-05-25 19:48:59 +02:00
if err != nil {
2020-09-18 08:19:34 +02:00
return reports , err
2020-05-25 19:48:59 +02:00
}
2021-06-15 14:53:42 +02:00
err , paths := verifyFFProjectCompliance ( config , sys , project , projectVersion , filterSet , influx , auditStatus )
reports = append ( reports , paths ... )
return reports , err
2020-09-18 08:19:34 +02:00
}
2021-02-26 14:43:03 +02:00
func classifyErrorOnLookup ( err error ) {
if strings . Contains ( err . Error ( ) , "connect: connection refused" ) || strings . Contains ( err . Error ( ) , "net/http: TLS handshake timeout" ) {
log . SetErrorCategory ( log . ErrorService )
}
}
2021-06-15 14:53:42 +02:00
func verifyFFProjectCompliance ( config fortifyExecuteScanOptions , sys fortify . System , project * models . Project , projectVersion * models . ProjectVersion , filterSet * models . FilterSet , influx * fortifyExecuteScanInflux , auditStatus map [ string ] string ) ( error , [ ] piperutils . Path ) {
reports := [ ] piperutils . Path { }
2020-05-25 19:48:59 +02:00
// Generate report
if config . Reporting {
2021-07-23 08:48:48 +02:00
resultURL := [ ] byte ( fmt . Sprintf ( "%v/html/ssc/version/%v/fix/null/" , config . ServerURL , projectVersion . ID ) )
2020-05-25 19:48:59 +02:00
ioutil . WriteFile ( fmt . Sprintf ( "%vtarget/%v-%v.%v" , config . ModulePath , * project . Name , * projectVersion . Name , "txt" ) , resultURL , 0700 )
data , err := generateAndDownloadQGateReport ( config , sys , project , projectVersion )
if err != nil {
2021-06-15 14:53:42 +02:00
return err , reports
2020-05-25 19:48:59 +02:00
}
ioutil . WriteFile ( fmt . Sprintf ( "%vtarget/%v-%v.%v" , config . ModulePath , * project . Name , * projectVersion . Name , config . ReportType ) , data , 0700 )
}
// Perform audit compliance checks
issueFilterSelectorSet , err := sys . GetIssueFilterSelectorOfProjectVersionByName ( projectVersion . ID , [ ] string { "Analysis" , "Folder" , "Category" } , nil )
if err != nil {
2021-06-15 14:53:42 +02:00
return errors . Wrapf ( err , "failed to fetch project version issue filter selector for project version ID %v" , projectVersion . ID ) , reports
2020-05-25 19:48:59 +02:00
}
log . Entry ( ) . Debugf ( "initial filter selector set: %v" , issueFilterSelectorSet )
2021-10-29 10:03:01 +02:00
spotChecksCountByCategory := [ ] fortify . SpotChecksAuditCount { }
numberOfViolations , issueGroups , err := analyseUnauditedIssues ( config , sys , projectVersion , filterSet , issueFilterSelectorSet , influx , auditStatus , & spotChecksCountByCategory )
2020-11-11 14:04:45 +02:00
if err != nil {
2021-06-15 14:53:42 +02:00
return errors . Wrap ( err , "failed to analyze unaudited issues" ) , reports
2020-11-11 14:04:45 +02:00
}
2021-08-10 10:49:31 +02:00
numberOfSuspiciousExploitable , issueGroupsSuspiciousExploitable := analyseSuspiciousExploitable ( config , sys , projectVersion , filterSet , issueFilterSelectorSet , influx , auditStatus )
numberOfViolations += numberOfSuspiciousExploitable
2021-06-15 14:53:42 +02:00
issueGroups = append ( issueGroups , issueGroupsSuspiciousExploitable ... )
2020-05-25 19:48:59 +02:00
log . Entry ( ) . Infof ( "Counted %v violations, details: %v" , numberOfViolations , auditStatus )
2020-09-18 08:19:34 +02:00
influx . fortify_data . fields . projectName = * project . Name
influx . fortify_data . fields . projectVersion = * projectVersion . Name
2021-08-10 10:49:31 +02:00
influx . fortify_data . fields . projectVersionID = projectVersion . ID
2020-10-22 11:40:42 +02:00
influx . fortify_data . fields . violations = numberOfViolations
2021-06-15 14:53:42 +02:00
2021-10-29 10:03:01 +02:00
fortifyReportingData := prepareReportData ( influx )
scanReport := fortify . CreateCustomReport ( fortifyReportingData , issueGroups )
2022-01-21 11:52:17 +02:00
paths , err := fortify . WriteCustomReports ( scanReport )
2021-10-29 10:03:01 +02:00
if err != nil {
return errors . Wrap ( err , "failed to write custom reports" ) , reports
}
reports = append ( reports , paths ... )
2022-01-21 11:52:17 +02:00
log . Entry ( ) . Debug ( "Checking whether GitHub issue creation/update is active" )
log . Entry ( ) . Debugf ( "%v, %v, %v, %v, %v, %v" , config . CreateResultIssue , numberOfViolations > 0 , len ( config . GithubToken ) > 0 , len ( config . GithubAPIURL ) > 0 , len ( config . Owner ) > 0 , len ( config . Repository ) > 0 )
if config . CreateResultIssue && numberOfViolations > 0 && len ( config . GithubToken ) > 0 && len ( config . GithubAPIURL ) > 0 && len ( config . Owner ) > 0 && len ( config . Repository ) > 0 {
log . Entry ( ) . Debug ( "Creating/updating GitHub issue with scan results" )
err = fortify . UploadReportToGithub ( scanReport , config . GithubToken , config . GithubAPIURL , config . Owner , config . Repository , config . Assignees )
if err != nil {
return errors . Wrap ( err , "failed to upload scan results into GitHub" ) , reports
}
}
2021-10-29 10:03:01 +02:00
jsonReport := fortify . CreateJSONReport ( fortifyReportingData , spotChecksCountByCategory , config . ServerURL )
paths , err = fortify . WriteJSONReport ( jsonReport )
if err != nil {
return errors . Wrap ( err , "failed to write json report" ) , reports
}
2021-06-15 14:53:42 +02:00
reports = append ( reports , paths ... )
2021-10-29 10:03:01 +02:00
2020-05-25 19:48:59 +02:00
if numberOfViolations > 0 {
2020-11-11 14:04:45 +02:00
log . SetErrorCategory ( log . ErrorCompliance )
2021-06-15 14:53:42 +02:00
return errors . New ( "fortify scan failed, the project is not compliant. For details check the archived report" ) , reports
2020-05-25 19:48:59 +02:00
}
2021-06-15 14:53:42 +02:00
return nil , reports
}
func prepareReportData ( influx * fortifyExecuteScanInflux ) fortify . FortifyReportData {
input := influx . fortify_data . fields
output := fortify . FortifyReportData { }
output . ProjectName = input . projectName
output . ProjectVersion = input . projectVersion
output . AuditAllAudited = input . auditAllAudited
output . AuditAllTotal = input . auditAllTotal
output . CorporateAudited = input . corporateAudited
output . CorporateTotal = input . corporateTotal
output . SpotChecksAudited = input . spotChecksAudited
output . SpotChecksGap = input . spotChecksGap
output . SpotChecksTotal = input . spotChecksTotal
output . Exploitable = input . exploitable
output . Suppressed = input . suppressed
output . Suspicious = input . suspicious
2021-10-29 10:03:01 +02:00
output . ProjectVersionID = input . projectVersionID
2022-01-21 11:52:17 +02:00
output . Violations = input . violations
2021-06-15 14:53:42 +02:00
return output
2020-05-25 19:48:59 +02:00
}
2021-10-29 10:03:01 +02:00
func analyseUnauditedIssues ( config fortifyExecuteScanOptions , sys fortify . System , projectVersion * models . ProjectVersion , filterSet * models . FilterSet , issueFilterSelectorSet * models . IssueFilterSelectorSet , influx * fortifyExecuteScanInflux , auditStatus map [ string ] string , spotChecksCountByCategory * [ ] fortify . SpotChecksAuditCount ) ( int , [ ] * models . ProjectVersionIssueGroup , error ) {
2020-05-25 19:48:59 +02:00
log . Entry ( ) . Info ( "Analyzing unaudited issues" )
reducedFilterSelectorSet := sys . ReduceIssueFilterSelectorSet ( issueFilterSelectorSet , [ ] string { "Folder" } , nil )
fetchedIssueGroups , err := sys . GetProjectIssuesByIDAndFilterSetGroupedBySelector ( projectVersion . ID , "" , filterSet . GUID , reducedFilterSelectorSet )
if err != nil {
2021-06-15 14:53:42 +02:00
return 0 , fetchedIssueGroups , errors . Wrapf ( err , "failed to fetch project version issue groups with filter set %v and selector %v for project version ID %v" , filterSet , issueFilterSelectorSet , projectVersion . ID )
2020-05-25 19:48:59 +02:00
}
overallViolations := 0
for _ , issueGroup := range fetchedIssueGroups {
2021-10-29 10:03:01 +02:00
issueDelta , err := getIssueDeltaFor ( config , sys , issueGroup , projectVersion . ID , filterSet , issueFilterSelectorSet , influx , auditStatus , spotChecksCountByCategory )
2020-11-11 14:04:45 +02:00
if err != nil {
2021-06-15 14:53:42 +02:00
return overallViolations , fetchedIssueGroups , errors . Wrap ( err , "failed to get issue delta" )
2020-11-11 14:04:45 +02:00
}
overallViolations += issueDelta
2020-05-25 19:48:59 +02:00
}
2021-06-15 14:53:42 +02:00
return overallViolations , fetchedIssueGroups , nil
2020-05-25 19:48:59 +02:00
}
2021-10-29 10:03:01 +02:00
func getIssueDeltaFor ( config fortifyExecuteScanOptions , sys fortify . System , issueGroup * models . ProjectVersionIssueGroup , projectVersionID int64 , filterSet * models . FilterSet , issueFilterSelectorSet * models . IssueFilterSelectorSet , influx * fortifyExecuteScanInflux , auditStatus map [ string ] string , spotChecksCountByCategory * [ ] fortify . SpotChecksAuditCount ) ( int , error ) {
2020-05-25 19:48:59 +02:00
totalMinusAuditedDelta := 0
group := ""
total := 0
audited := 0
if issueGroup != nil {
group = * issueGroup . ID
total = int ( * issueGroup . TotalCount )
audited = int ( * issueGroup . AuditedCount )
}
groupTotalMinusAuditedDelta := total - audited
if groupTotalMinusAuditedDelta > 0 {
reducedFilterSelectorSet := sys . ReduceIssueFilterSelectorSet ( issueFilterSelectorSet , [ ] string { "Folder" , "Analysis" } , [ ] string { group } )
folderSelector := sys . GetFilterSetByDisplayName ( reducedFilterSelectorSet , "Folder" )
if folderSelector == nil {
2020-11-11 14:04:45 +02:00
return totalMinusAuditedDelta , fmt . Errorf ( "folder selector not found" )
2020-05-25 19:48:59 +02:00
}
analysisSelector := sys . GetFilterSetByDisplayName ( reducedFilterSelectorSet , "Analysis" )
auditStatus [ group ] = fmt . Sprintf ( "%v total : %v audited" , total , audited )
if strings . Contains ( config . MustAuditIssueGroups , group ) {
totalMinusAuditedDelta += groupTotalMinusAuditedDelta
if group == "Corporate Security Requirements" {
2020-10-22 11:40:42 +02:00
influx . fortify_data . fields . corporateTotal = total
influx . fortify_data . fields . corporateAudited = audited
2020-05-25 19:48:59 +02:00
}
if group == "Audit All" {
2020-10-22 11:40:42 +02:00
influx . fortify_data . fields . auditAllTotal = total
influx . fortify_data . fields . auditAllAudited = audited
2020-05-25 19:48:59 +02:00
}
log . Entry ( ) . Errorf ( "[projectVersionId %v]: Unaudited %v detected, count %v" , projectVersionID , group , totalMinusAuditedDelta )
logIssueURL ( config , projectVersionID , folderSelector , analysisSelector )
}
if strings . Contains ( config . SpotAuditIssueGroups , group ) {
log . Entry ( ) . Infof ( "Analyzing %v" , config . SpotAuditIssueGroups )
filter := fmt . Sprintf ( "%v:%v" , folderSelector . EntityType , folderSelector . SelectorOptions [ 0 ] . Value )
fetchedIssueGroups , err := sys . GetProjectIssuesByIDAndFilterSetGroupedBySelector ( projectVersionID , filter , filterSet . GUID , sys . ReduceIssueFilterSelectorSet ( issueFilterSelectorSet , [ ] string { "Category" } , nil ) )
if err != nil {
2020-11-11 14:04:45 +02:00
return totalMinusAuditedDelta , errors . Wrapf ( err , "failed to fetch project version issue groups with filter %v, filter set %v and selector %v for project version ID %v" , filter , filterSet , issueFilterSelectorSet , projectVersionID )
2020-05-25 19:48:59 +02:00
}
2021-10-29 10:03:01 +02:00
totalMinusAuditedDelta += getSpotIssueCount ( config , sys , fetchedIssueGroups , projectVersionID , filterSet , reducedFilterSelectorSet , influx , auditStatus , spotChecksCountByCategory )
2020-05-25 19:48:59 +02:00
}
}
2020-11-11 14:04:45 +02:00
return totalMinusAuditedDelta , nil
2020-05-25 19:48:59 +02:00
}
2021-10-29 10:03:01 +02:00
func getSpotIssueCount ( config fortifyExecuteScanOptions , sys fortify . System , spotCheckCategories [ ] * models . ProjectVersionIssueGroup , projectVersionID int64 , filterSet * models . FilterSet , issueFilterSelectorSet * models . IssueFilterSelectorSet , influx * fortifyExecuteScanInflux , auditStatus map [ string ] string , spotChecksCountByCategory * [ ] fortify . SpotChecksAuditCount ) int {
2020-05-25 19:48:59 +02:00
overallDelta := 0
overallIssues := 0
overallIssuesAudited := 0
for _ , issueGroup := range spotCheckCategories {
group := ""
total := 0
audited := 0
if issueGroup != nil {
group = * issueGroup . ID
total = int ( * issueGroup . TotalCount )
audited = int ( * issueGroup . AuditedCount )
}
flagOutput := ""
if ( ( total <= config . SpotCheckMinimum || config . SpotCheckMinimum < 0 ) && audited != total ) || ( total > config . SpotCheckMinimum && audited < config . SpotCheckMinimum ) {
currentDelta := config . SpotCheckMinimum - audited
if config . SpotCheckMinimum < 0 || config . SpotCheckMinimum > total {
currentDelta = total - audited
}
if currentDelta > 0 {
filterSelectorFolder := sys . GetFilterSetByDisplayName ( issueFilterSelectorSet , "Folder" )
filterSelectorAnalysis := sys . GetFilterSetByDisplayName ( issueFilterSelectorSet , "Analysis" )
overallDelta += currentDelta
log . Entry ( ) . Errorf ( "[projectVersionId %v]: %v unaudited spot check issues detected in group %v" , projectVersionID , currentDelta , group )
logIssueURL ( config , projectVersionID , filterSelectorFolder , filterSelectorAnalysis )
flagOutput = checkString
}
}
overallIssues += total
overallIssuesAudited += audited
auditStatus [ group ] = fmt . Sprintf ( "%v total : %v audited %v" , total , audited , flagOutput )
2021-10-29 10:03:01 +02:00
* spotChecksCountByCategory = append ( * spotChecksCountByCategory , fortify . SpotChecksAuditCount { Audited : audited , Total : total , Type : group } )
2020-05-25 19:48:59 +02:00
}
2020-10-22 11:40:42 +02:00
influx . fortify_data . fields . spotChecksTotal = overallIssues
influx . fortify_data . fields . spotChecksAudited = overallIssuesAudited
influx . fortify_data . fields . spotChecksGap = overallDelta
2020-05-25 19:48:59 +02:00
return overallDelta
}
2021-06-15 14:53:42 +02:00
func analyseSuspiciousExploitable ( config fortifyExecuteScanOptions , sys fortify . System , projectVersion * models . ProjectVersion , filterSet * models . FilterSet , issueFilterSelectorSet * models . IssueFilterSelectorSet , influx * fortifyExecuteScanInflux , auditStatus map [ string ] string ) ( int , [ ] * models . ProjectVersionIssueGroup ) {
2020-05-25 19:48:59 +02:00
log . Entry ( ) . Info ( "Analyzing suspicious and exploitable issues" )
reducedFilterSelectorSet := sys . ReduceIssueFilterSelectorSet ( issueFilterSelectorSet , [ ] string { "Analysis" } , [ ] string { } )
fetchedGroups , err := sys . GetProjectIssuesByIDAndFilterSetGroupedBySelector ( projectVersion . ID , "" , filterSet . GUID , reducedFilterSelectorSet )
suspiciousCount := 0
exploitableCount := 0
for _ , issueGroup := range fetchedGroups {
if * issueGroup . ID == "3" {
suspiciousCount = int ( * issueGroup . TotalCount )
} else if * issueGroup . ID == "4" {
exploitableCount = int ( * issueGroup . TotalCount )
}
}
result := 0
if ( suspiciousCount > 0 && config . ConsiderSuspicious ) || exploitableCount > 0 {
result = result + suspiciousCount + exploitableCount
log . Entry ( ) . Errorf ( "[projectVersionId %v]: %v suspicious and %v exploitable issues detected" , projectVersion . ID , suspiciousCount , exploitableCount )
log . Entry ( ) . Errorf ( "%v/html/ssc/index.jsp#!/version/%v/fix?issueGrouping=%v_%v&issueFilters=%v_%v" , config . ServerURL , projectVersion . ID , reducedFilterSelectorSet . GroupBySet [ 0 ] . EntityType , reducedFilterSelectorSet . GroupBySet [ 0 ] . Value , reducedFilterSelectorSet . FilterBySet [ 0 ] . EntityType , reducedFilterSelectorSet . FilterBySet [ 0 ] . Value )
}
issueStatistics , err := sys . GetIssueStatisticsOfProjectVersion ( projectVersion . ID )
if err != nil {
log . Entry ( ) . WithError ( err ) . Errorf ( "Failed to fetch project version statistics for project version ID %v" , projectVersion . ID )
}
auditStatus [ "Suspicious" ] = fmt . Sprintf ( "%v" , suspiciousCount )
auditStatus [ "Exploitable" ] = fmt . Sprintf ( "%v" , exploitableCount )
suppressedCount := * issueStatistics [ 0 ] . SuppressedCount
if suppressedCount > 0 {
auditStatus [ "Suppressed" ] = fmt . Sprintf ( "WARNING: Detected %v suppressed issues which could violate audit compliance!!!" , suppressedCount )
}
2020-10-22 11:40:42 +02:00
influx . fortify_data . fields . suspicious = suspiciousCount
influx . fortify_data . fields . exploitable = exploitableCount
influx . fortify_data . fields . suppressed = int ( suppressedCount )
2020-05-25 19:48:59 +02:00
2021-06-15 14:53:42 +02:00
return result , fetchedGroups
2020-05-25 19:48:59 +02:00
}
func logIssueURL ( config fortifyExecuteScanOptions , projectVersionID int64 , folderSelector , analysisSelector * models . IssueFilterSelector ) {
url := fmt . Sprintf ( "%v/html/ssc/index.jsp#!/version/%v/fix" , config . ServerURL , projectVersionID )
if len ( folderSelector . SelectorOptions ) > 0 {
url += fmt . Sprintf ( "?issueFilters=%v_%v:%v" ,
folderSelector . EntityType ,
folderSelector . Value ,
folderSelector . SelectorOptions [ 0 ] . Value )
} else {
log . Entry ( ) . Debugf ( "no 'filter by set' array entries" )
}
if analysisSelector != nil {
url += fmt . Sprintf ( "&issueFilters=%v_%v:" ,
analysisSelector . EntityType ,
analysisSelector . Value )
} else {
log . Entry ( ) . Debugf ( "no second entry in 'filter by set' array" )
}
log . Entry ( ) . Error ( url )
}
func generateAndDownloadQGateReport ( config fortifyExecuteScanOptions , sys fortify . System , project * models . Project , projectVersion * models . ProjectVersion ) ( [ ] byte , error ) {
log . Entry ( ) . Infof ( "Generating report with template ID %v" , config . ReportTemplateID )
report , err := sys . GenerateQGateReport ( project . ID , projectVersion . ID , int64 ( config . ReportTemplateID ) , * project . Name , * projectVersion . Name , config . ReportType )
if err != nil {
2020-11-11 14:04:45 +02:00
return [ ] byte { } , errors . Wrap ( err , "failed to generate Q-Gate report" )
2020-05-25 19:48:59 +02:00
}
log . Entry ( ) . Debugf ( "Triggered report generation of report ID %v" , report . ID )
status := report . Status
2020-09-29 18:26:16 +02:00
for status == "PROCESSING" || status == "SCHED_PROCESSING" {
2020-05-25 19:48:59 +02:00
time . Sleep ( 10 * time . Second )
report , err = sys . GetReportDetails ( report . ID )
if err != nil {
return [ ] byte { } , fmt . Errorf ( "Failed to fetch Q-Gate report generation status: %w" , err )
}
status = report . Status
}
2020-10-27 14:12:31 +02:00
data , err := sys . DownloadReportFile ( config . ReportDownloadEndpoint , report . ID )
2020-05-25 19:48:59 +02:00
if err != nil {
return [ ] byte { } , fmt . Errorf ( "Failed to download Q-Gate Report: %w" , err )
}
return data , nil
}
2020-08-11 15:29:00 +02:00
var errProcessing = errors . New ( "artifact still processing" )
func checkArtifactStatus ( config fortifyExecuteScanOptions , projectVersionID int64 , filterSet * models . FilterSet , artifact * models . Artifact , retries int , pollingDelay , timeout time . Duration ) error {
2020-05-25 19:48:59 +02:00
if "PROCESSING" == artifact . Status || "SCHED_PROCESSING" == artifact . Status {
2020-08-11 15:29:00 +02:00
pollingTime := time . Duration ( retries ) * pollingDelay
if pollingTime >= timeout {
2021-02-26 14:43:03 +02:00
log . SetErrorCategory ( log . ErrorService )
2020-08-11 15:29:00 +02:00
return fmt . Errorf ( "terminating after %v since artifact for Project Version %v is still in status %v" , timeout , projectVersionID , artifact . Status )
2020-05-25 19:48:59 +02:00
}
log . Entry ( ) . Infof ( "Most recent artifact uploaded on %v of Project Version %v is still in status %v..." , artifact . UploadDate , projectVersionID , artifact . Status )
2020-08-11 15:29:00 +02:00
time . Sleep ( pollingDelay )
return errProcessing
2020-05-25 19:48:59 +02:00
}
if "REQUIRE_AUTH" == artifact . Status {
// verify no manual issue approval needed
2021-02-26 14:43:03 +02:00
log . SetErrorCategory ( log . ErrorCompliance )
return fmt . Errorf ( "There are artifacts that require manual approval for Project Version %v, please visit Fortify SSC and approve them for processing\n%v/html/ssc/index.jsp#!/version/%v/artifacts?filterSet=%v" , projectVersionID , config . ServerURL , projectVersionID , filterSet . GUID )
2020-05-25 19:48:59 +02:00
}
if "ERROR_PROCESSING" == artifact . Status {
2021-02-26 14:43:03 +02:00
log . SetErrorCategory ( log . ErrorService )
2020-05-25 19:48:59 +02:00
return fmt . Errorf ( "There are artifacts that failed processing for Project Version %v\n%v/html/ssc/index.jsp#!/version/%v/artifacts?filterSet=%v" , projectVersionID , config . ServerURL , projectVersionID , filterSet . GUID )
}
return nil
}
2020-08-11 15:29:00 +02:00
func verifyScanResultsFinishedUploading ( config fortifyExecuteScanOptions , sys fortify . System , projectVersionID int64 , buildLabel string , filterSet * models . FilterSet , pollingDelay , timeout time . Duration ) error {
2020-05-25 19:48:59 +02:00
log . Entry ( ) . Debug ( "Verifying scan results have finished uploading and processing" )
var artifacts [ ] * models . Artifact
var relatedUpload * models . Artifact
2020-08-11 15:29:00 +02:00
var err error
retries := 0
2020-05-25 19:48:59 +02:00
for relatedUpload == nil {
2020-08-11 15:29:00 +02:00
artifacts , err = sys . GetArtifactsOfProjectVersion ( projectVersionID )
2020-05-25 19:48:59 +02:00
log . Entry ( ) . Debugf ( "Received %v artifacts for project version ID %v" , len ( artifacts ) , projectVersionID )
if err != nil {
2020-08-11 15:29:00 +02:00
return fmt . Errorf ( "failed to fetch artifacts of project version ID %v" , projectVersionID )
2020-05-25 19:48:59 +02:00
}
2020-08-11 15:29:00 +02:00
if len ( artifacts ) == 0 {
return fmt . Errorf ( "no uploaded artifacts for assessment detected for project version with ID %v" , projectVersionID )
}
latest := artifacts [ 0 ]
err = checkArtifactStatus ( config , projectVersionID , filterSet , latest , retries , pollingDelay , timeout )
if err != nil {
if err == errProcessing {
retries ++
continue
2020-05-25 19:48:59 +02:00
}
2020-08-11 15:29:00 +02:00
return err
2020-05-25 19:48:59 +02:00
}
2020-08-11 15:29:00 +02:00
relatedUpload = findArtifactByBuildLabel ( artifacts , buildLabel )
2020-05-25 19:48:59 +02:00
if relatedUpload == nil {
log . Entry ( ) . Warn ( "Unable to identify artifact based on the build label, will consider most recent artifact as related to the scan" )
relatedUpload = artifacts [ 0 ]
}
}
differenceInSeconds := calculateTimeDifferenceToLastUpload ( relatedUpload . UploadDate , projectVersionID )
// Use the absolute value for checking the time difference
if differenceInSeconds > float64 ( 60 * config . DeltaMinutes ) {
2020-08-11 15:29:00 +02:00
return errors . New ( "no recent upload detected on Project Version" )
2020-05-25 19:48:59 +02:00
}
for _ , upload := range artifacts {
if upload . Status == "ERROR_PROCESSING" {
2020-08-11 15:29:00 +02:00
log . Entry ( ) . Warn ( "Previous uploads detected that failed processing, please ensure that your scans are properly configured" )
break
2020-05-25 19:48:59 +02:00
}
}
2020-08-11 15:29:00 +02:00
return nil
}
func findArtifactByBuildLabel ( artifacts [ ] * models . Artifact , buildLabel string ) * models . Artifact {
if len ( buildLabel ) == 0 {
return nil
}
for _ , artifact := range artifacts {
if len ( buildLabel ) > 0 && artifact . Embed != nil && artifact . Embed . Scans != nil && len ( artifact . Embed . Scans ) > 0 {
scan := artifact . Embed . Scans [ 0 ]
if scan != nil && strings . HasSuffix ( scan . BuildLabel , buildLabel ) {
return artifact
}
}
2020-05-25 19:48:59 +02:00
}
return nil
}
func calculateTimeDifferenceToLastUpload ( uploadDate models . Iso8601MilliDateTime , projectVersionID int64 ) float64 {
log . Entry ( ) . Infof ( "Last upload on project version %v happened on %v" , projectVersionID , uploadDate )
uploadDateAsTime := time . Time ( uploadDate )
duration := time . Since ( uploadDateAsTime )
log . Entry ( ) . Debugf ( "Difference duration is %v" , duration )
absoluteSeconds := math . Abs ( duration . Seconds ( ) )
log . Entry ( ) . Infof ( "Difference since %v in seconds is %v" , uploadDateAsTime , absoluteSeconds )
return absoluteSeconds
}
2020-11-11 14:04:45 +02:00
func executeTemplatedCommand ( utils fortifyUtils , cmdTemplate [ ] string , context map [ string ] string ) error {
2020-05-25 19:48:59 +02:00
for index , cmdTemplatePart := range cmdTemplate {
result , err := piperutils . ExecuteTemplate ( cmdTemplatePart , context )
if err != nil {
2020-11-11 14:04:45 +02:00
return errors . Wrapf ( err , "failed to transform template for command fragment: %v" , cmdTemplatePart )
2020-05-25 19:48:59 +02:00
}
cmdTemplate [ index ] = result
}
2020-11-10 18:14:55 +02:00
err := utils . RunExecutable ( cmdTemplate [ 0 ] , cmdTemplate [ 1 : ] ... )
2020-05-25 19:48:59 +02:00
if err != nil {
2020-11-11 14:04:45 +02:00
return errors . Wrapf ( err , "failed to execute command %v" , cmdTemplate )
2020-05-25 19:48:59 +02:00
}
2020-11-11 14:04:45 +02:00
return nil
2020-05-25 19:48:59 +02:00
}
2020-11-11 14:04:45 +02:00
func autoresolvePipClasspath ( executable string , parameters [ ] string , file string , utils fortifyUtils ) ( string , error ) {
2020-05-25 19:48:59 +02:00
// redirect stdout and create cp file from command output
outfile , err := os . Create ( file )
if err != nil {
2020-11-11 14:04:45 +02:00
return "" , errors . Wrapf ( err , "failed to create classpath file" )
2020-05-25 19:48:59 +02:00
}
defer outfile . Close ( )
2020-11-10 18:14:55 +02:00
utils . Stdout ( outfile )
err = utils . RunExecutable ( executable , parameters ... )
2020-05-25 19:48:59 +02:00
if err != nil {
2020-11-11 14:04:45 +02:00
return "" , errors . Wrapf ( err , "failed to run classpath autodetection command %v with parameters %v" , executable , parameters )
2020-05-25 19:48:59 +02:00
}
2020-11-10 18:14:55 +02:00
utils . Stdout ( log . Entry ( ) . Writer ( ) )
2020-11-11 14:04:45 +02:00
return readClasspathFile ( file ) , nil
2020-05-25 19:48:59 +02:00
}
2021-06-16 08:15:41 +02:00
func autoresolveMavenClasspath ( config fortifyExecuteScanOptions , file string , utils fortifyUtils ) ( string , error ) {
2020-05-29 15:42:35 +02:00
if filepath . IsAbs ( file ) {
log . Entry ( ) . Warnf ( "Passing an absolute path for -Dmdep.outputFile results in the classpath only for the last module in multi-module maven projects." )
}
2021-06-28 12:40:20 +02:00
defines := generateMavenFortifyDefines ( & config , file )
2020-05-25 19:48:59 +02:00
executeOptions := maven . ExecuteOptions {
2020-05-29 15:42:35 +02:00
PomPath : config . BuildDescriptorFile ,
ProjectSettingsFile : config . ProjectSettingsFile ,
GlobalSettingsFile : config . GlobalSettingsFile ,
M2Path : config . M2Path ,
2021-06-28 12:40:20 +02:00
Goals : [ ] string { "dependency:build-classpath" , "package" } ,
Defines : defines ,
2020-05-29 15:42:35 +02:00
ReturnStdout : false ,
}
2020-11-10 18:14:55 +02:00
_ , err := maven . Execute ( & executeOptions , utils )
2020-05-25 19:48:59 +02:00
if err != nil {
2021-06-28 12:40:20 +02:00
log . Entry ( ) . WithError ( err ) . Warnf ( "failed to determine classpath using Maven: %v" , err )
2020-05-25 19:48:59 +02:00
}
2021-06-16 08:15:41 +02:00
return readAllClasspathFiles ( file ) , nil
2020-05-29 15:42:35 +02:00
}
2021-06-28 12:40:20 +02:00
func generateMavenFortifyDefines ( config * fortifyExecuteScanOptions , file string ) [ ] string {
defines := [ ] string {
fmt . Sprintf ( "-Dmdep.outputFile=%v" , file ) ,
2021-07-02 09:43:34 +02:00
// Parameter to indicate to maven build that the fortify step is the trigger, can be used for optimizations
"-Dfortify" ,
2021-06-28 12:40:20 +02:00
"-DincludeScope=compile" ,
"-DskipTests" ,
2021-07-02 09:43:34 +02:00
"-Dmaven.javadoc.skip=true" ,
2021-06-28 12:40:20 +02:00
"--fail-at-end" }
if len ( config . BuildDescriptorExcludeList ) > 0 {
// From the documentation, these are file paths to a module's pom.xml.
// For MTA projects, we support pom.xml files here and skip others.
for _ , exclude := range config . BuildDescriptorExcludeList {
if ! strings . HasSuffix ( exclude , "pom.xml" ) {
continue
}
exists , _ := piperutils . FileExists ( exclude )
if ! exists {
continue
}
moduleName := filepath . Dir ( exclude )
if moduleName != "" {
defines = append ( defines , "-pl" , "!" + moduleName )
}
}
}
return defines
}
2020-05-29 15:42:35 +02:00
// readAllClasspathFiles tests whether the passed file is an absolute path. If not, it will glob for
// all files under the current directory with the given file name and concatenate their contents.
// Otherwise it will return the contents pointed to by the absolute path.
func readAllClasspathFiles ( file string ) string {
var paths [ ] string
if filepath . IsAbs ( file ) {
paths = [ ] string { file }
} else {
paths , _ = doublestar . Glob ( filepath . Join ( "**" , file ) )
log . Entry ( ) . Debugf ( "Concatenating the class paths from %v" , paths )
}
var contents string
const separator = ":"
for _ , path := range paths {
contents += separator + readClasspathFile ( path )
}
return removeDuplicates ( contents , separator )
2020-05-25 19:48:59 +02:00
}
func readClasspathFile ( file string ) string {
data , err := ioutil . ReadFile ( file )
if err != nil {
log . Entry ( ) . WithError ( err ) . Warnf ( "failed to read classpath from file '%v'" , file )
}
2020-05-29 15:42:35 +02:00
result := strings . TrimSpace ( string ( data ) )
if len ( result ) == 0 {
log . Entry ( ) . Warnf ( "classpath from file '%v' was empty" , file )
}
return result
}
func removeDuplicates ( contents , separator string ) string {
if separator == "" || contents == "" {
return contents
}
entries := strings . Split ( contents , separator )
entrySet := map [ string ] struct { } { }
contents = ""
for _ , entry := range entries {
if entry == "" {
continue
}
_ , contained := entrySet [ entry ]
if ! contained {
entrySet [ entry ] = struct { } { }
contents += entry + separator
}
}
if contents != "" {
// Remove trailing "separator"
contents = contents [ : len ( contents ) - len ( separator ) ]
}
return contents
2020-05-25 19:48:59 +02:00
}
2020-11-11 14:04:45 +02:00
func triggerFortifyScan ( config fortifyExecuteScanOptions , utils fortifyUtils , buildID , buildLabel , buildProject string ) error {
2020-05-27 11:45:01 +02:00
var err error = nil
2020-05-25 19:48:59 +02:00
// Do special Python related prep
pipVersion := "pip3"
if config . PythonVersion != "python3" {
pipVersion = "pip2"
}
classpath := ""
if config . BuildTool == "maven" {
if config . AutodetectClasspath {
2021-06-16 08:15:41 +02:00
classpath , err = autoresolveMavenClasspath ( config , classpathFileName , utils )
if err != nil {
return err
}
2020-05-25 19:48:59 +02:00
}
2020-05-27 11:45:01 +02:00
config . Translate , err = populateMavenTranslate ( & config , classpath )
if err != nil {
log . Entry ( ) . WithError ( err ) . Warnf ( "failed to apply src ('%s') or exclude ('%s') parameter" , config . Src , config . Exclude )
2020-05-25 19:48:59 +02:00
}
2021-03-25 10:59:49 +02:00
} else if config . BuildTool == "pip" {
2020-05-25 19:48:59 +02:00
if config . AutodetectClasspath {
2020-06-02 13:47:07 +02:00
separator := getSeparator ( )
script := fmt . Sprintf ( "import sys;p=sys.path;p.remove('');print('%v'.join(p))" , separator )
2020-11-11 14:04:45 +02:00
classpath , err = autoresolvePipClasspath ( config . PythonVersion , [ ] string { "-c" , script } , classpathFileName , utils )
if err != nil {
return errors . Wrap ( err , "failed to autoresolve pip classpath" )
}
2020-05-25 19:48:59 +02:00
}
// install the dev dependencies
if len ( config . PythonRequirementsFile ) > 0 {
context := map [ string ] string { }
cmdTemplate := [ ] string { pipVersion , "install" , "--user" , "-r" , config . PythonRequirementsFile }
cmdTemplate = append ( cmdTemplate , tokenize ( config . PythonRequirementsInstallSuffix ) ... )
2020-11-10 18:14:55 +02:00
executeTemplatedCommand ( utils , cmdTemplate , context )
2020-05-25 19:48:59 +02:00
}
2020-11-10 18:14:55 +02:00
executeTemplatedCommand ( utils , tokenize ( config . PythonInstallCommand ) , map [ string ] string { "Pip" : pipVersion } )
2020-05-25 19:48:59 +02:00
2020-05-27 11:45:01 +02:00
config . Translate , err = populatePipTranslate ( & config , classpath )
if err != nil {
2020-06-02 13:47:07 +02:00
log . Entry ( ) . WithError ( err ) . Warnf ( "failed to apply pythonAdditionalPath ('%s') or src ('%s') parameter" , config . PythonAdditionalPath , config . Src )
2020-05-25 19:48:59 +02:00
}
2020-05-27 11:45:01 +02:00
2021-03-25 10:59:49 +02:00
} else {
return fmt . Errorf ( "buildTool '%s' is not supported by this step" , config . BuildTool )
2020-05-25 19:48:59 +02:00
}
2022-01-21 11:52:17 +02:00
err = translateProject ( & config , utils , buildID , classpath )
if err != nil {
return err
}
2020-05-25 19:48:59 +02:00
2020-11-11 14:04:45 +02:00
return scanProject ( & config , utils , buildID , buildLabel , buildProject )
2020-05-25 19:48:59 +02:00
}
2020-05-27 11:45:01 +02:00
func populatePipTranslate ( config * fortifyExecuteScanOptions , classpath string ) ( string , error ) {
if len ( config . Translate ) > 0 {
return config . Translate , nil
}
var translateList [ ] map [ string ] interface { }
translateList = append ( translateList , make ( map [ string ] interface { } ) )
2020-06-02 13:47:07 +02:00
separator := getSeparator ( )
translateList [ 0 ] [ "pythonPath" ] = classpath + separator +
getSuppliedOrDefaultListAsString ( config . PythonAdditionalPath , [ ] string { } , separator )
translateList [ 0 ] [ "src" ] = getSuppliedOrDefaultListAsString (
config . Src , [ ] string { "./**/*" } , ":" )
translateList [ 0 ] [ "exclude" ] = getSuppliedOrDefaultListAsString (
config . Exclude , [ ] string { "./**/tests/**/*" , "./**/setup.py" } , separator )
2020-05-27 11:45:01 +02:00
translateJSON , err := json . Marshal ( translateList )
return string ( translateJSON ) , err
}
func populateMavenTranslate ( config * fortifyExecuteScanOptions , classpath string ) ( string , error ) {
if len ( config . Translate ) > 0 {
return config . Translate , nil
}
var translateList [ ] map [ string ] interface { }
translateList = append ( translateList , make ( map [ string ] interface { } ) )
translateList [ 0 ] [ "classpath" ] = classpath
2020-06-02 13:47:07 +02:00
setTranslateEntryIfNotEmpty ( translateList [ 0 ] , "src" , ":" , config . Src ,
2021-07-02 09:43:34 +02:00
[ ] string { "**/*.xml" , "**/*.html" , "**/*.jsp" , "**/*.js" , "**/src/main/resources/**/*" , "**/src/main/java/**/*" , "**/target/main/java/**/*" , "**/target/main/resources/**/*" , "**/target/generated-sources/**/*" } )
2020-06-02 13:47:07 +02:00
2021-07-09 10:19:42 +02:00
setTranslateEntryIfNotEmpty ( translateList [ 0 ] , "exclude" , getSeparator ( ) , config . Exclude , [ ] string { "**/src/test/**/*" } )
2020-05-27 11:45:01 +02:00
translateJSON , err := json . Marshal ( translateList )
return string ( translateJSON ) , err
}
2022-01-21 11:52:17 +02:00
func translateProject ( config * fortifyExecuteScanOptions , utils fortifyUtils , buildID , classpath string ) error {
2020-05-25 19:48:59 +02:00
var translateList [ ] map [ string ] string
json . Unmarshal ( [ ] byte ( config . Translate ) , & translateList )
log . Entry ( ) . Debugf ( "Translating with options: %v" , translateList )
for _ , translate := range translateList {
if len ( classpath ) > 0 {
translate [ "autoClasspath" ] = classpath
}
2022-01-21 11:52:17 +02:00
err := handleSingleTranslate ( config , utils , buildID , translate )
if err != nil {
return err
}
2020-05-25 19:48:59 +02:00
}
2022-01-21 11:52:17 +02:00
return nil
2020-05-25 19:48:59 +02:00
}
2020-11-11 14:04:45 +02:00
func handleSingleTranslate ( config * fortifyExecuteScanOptions , command fortifyUtils , buildID string , t map [ string ] string ) error {
2020-05-25 19:48:59 +02:00
if t != nil {
log . Entry ( ) . Debugf ( "Handling translate config %v" , t )
translateOptions := [ ] string {
"-verbose" ,
"-64" ,
"-b" ,
buildID ,
}
translateOptions = append ( translateOptions , tokenize ( config . Memory ) ... )
translateOptions = appendToOptions ( config , translateOptions , t )
log . Entry ( ) . Debugf ( "Running sourceanalyzer translate command with options %v" , translateOptions )
err := command . RunExecutable ( "sourceanalyzer" , translateOptions ... )
if err != nil {
2020-11-11 14:04:45 +02:00
return errors . Wrapf ( err , "failed to execute sourceanalyzer translate command with options %v" , translateOptions )
2020-05-25 19:48:59 +02:00
}
} else {
log . Entry ( ) . Debug ( "Skipping translate with nil value" )
}
2020-11-11 14:04:45 +02:00
return nil
2020-05-25 19:48:59 +02:00
}
2020-11-11 14:04:45 +02:00
func scanProject ( config * fortifyExecuteScanOptions , command fortifyUtils , buildID , buildLabel , buildProject string ) error {
2020-05-25 19:48:59 +02:00
var scanOptions = [ ] string {
"-verbose" ,
"-64" ,
"-b" ,
buildID ,
"-scan" ,
}
scanOptions = append ( scanOptions , tokenize ( config . Memory ) ... )
if config . QuickScan {
scanOptions = append ( scanOptions , "-quick" )
}
2021-03-09 14:16:21 +02:00
if len ( config . AdditionalScanParameters ) > 0 {
for _ , scanParameter := range config . AdditionalScanParameters {
2021-03-09 14:41:07 +02:00
scanOptions = append ( scanOptions , scanParameter )
2021-03-09 14:16:21 +02:00
}
}
2020-05-25 19:48:59 +02:00
if len ( buildLabel ) > 0 {
scanOptions = append ( scanOptions , "-build-label" , buildLabel )
}
2020-05-28 10:45:06 +02:00
if len ( buildProject ) > 0 {
scanOptions = append ( scanOptions , "-build-project" , buildProject )
}
2020-05-25 19:48:59 +02:00
scanOptions = append ( scanOptions , "-logfile" , "target/fortify-scan.log" , "-f" , "target/result.fpr" )
err := command . RunExecutable ( "sourceanalyzer" , scanOptions ... )
if err != nil {
2020-11-11 14:04:45 +02:00
return errors . Wrapf ( err , "failed to execute sourceanalyzer scan command with scanOptions %v" , scanOptions )
2020-05-25 19:48:59 +02:00
}
2020-11-11 14:04:45 +02:00
return nil
2020-05-25 19:48:59 +02:00
}
2022-01-21 11:52:17 +02:00
func determinePullRequestMerge ( config fortifyExecuteScanOptions ) ( string , string ) {
author := ""
2020-05-25 19:48:59 +02:00
ctx , client , err := piperGithub . NewClient ( config . GithubToken , config . GithubAPIURL , "" )
if err == nil {
2022-01-21 11:52:17 +02:00
prID , author , err := determinePullRequestMergeGithub ( ctx , config , client . PullRequests )
2020-05-25 19:48:59 +02:00
if err != nil {
log . Entry ( ) . WithError ( err ) . Warn ( "Failed to get PR metadata via GitHub client" )
} else {
2022-01-21 11:52:17 +02:00
return prID , author
2020-05-25 19:48:59 +02:00
}
}
log . Entry ( ) . Infof ( "Trying to determine PR ID in commit message: %v" , config . CommitMessage )
r , _ := regexp . Compile ( config . PullRequestMessageRegex )
matches := r . FindSubmatch ( [ ] byte ( config . CommitMessage ) )
if matches != nil && len ( matches ) > 1 {
2022-01-21 11:52:17 +02:00
return string ( matches [ config . PullRequestMessageRegexGroup ] ) , author
2020-05-25 19:48:59 +02:00
}
2022-01-24 12:59:33 +02:00
return "0" , ""
2020-05-25 19:48:59 +02:00
}
2022-01-21 11:52:17 +02:00
func determinePullRequestMergeGithub ( ctx context . Context , config fortifyExecuteScanOptions , pullRequestServiceInstance pullRequestService ) ( string , string , error ) {
2022-01-24 12:59:33 +02:00
number := "0"
email := ""
2020-05-25 19:48:59 +02:00
options := github . PullRequestListOptions { State : "closed" , Sort : "updated" , Direction : "desc" }
prList , _ , err := pullRequestServiceInstance . ListPullRequestsWithCommit ( ctx , config . Owner , config . Repository , config . CommitID , & options )
if err == nil && len ( prList ) > 0 {
2022-01-24 12:59:33 +02:00
number = fmt . Sprintf ( "%v" , prList [ 0 ] . GetNumber ( ) )
if nil != prList [ 0 ] . User {
email = * ( prList [ 0 ] . User . Email )
}
return number , email , nil
2020-05-25 19:48:59 +02:00
}
2022-01-24 12:59:33 +02:00
return number , email , err
2020-05-25 19:48:59 +02:00
}
func appendToOptions ( config * fortifyExecuteScanOptions , options [ ] string , t map [ string ] string ) [ ] string {
2020-06-02 13:47:07 +02:00
switch config . BuildTool {
case "windows" :
2020-05-25 19:48:59 +02:00
if len ( t [ "aspnetcore" ] ) > 0 {
options = append ( options , "-aspnetcore" )
}
if len ( t [ "dotNetCoreVersion" ] ) > 0 {
options = append ( options , "-dotnet-core-version" , t [ "dotNetCoreVersion" ] )
}
if len ( t [ "libDirs" ] ) > 0 {
options = append ( options , "-libdirs" , t [ "libDirs" ] )
}
2020-06-02 13:47:07 +02:00
case "maven" :
2020-05-25 19:48:59 +02:00
if len ( t [ "autoClasspath" ] ) > 0 {
options = append ( options , "-cp" , t [ "autoClasspath" ] )
} else if len ( t [ "classpath" ] ) > 0 {
options = append ( options , "-cp" , t [ "classpath" ] )
2020-05-29 15:42:35 +02:00
} else {
log . Entry ( ) . Debugf ( "no field 'autoClasspath' or 'classpath' in map or both empty" )
2020-05-25 19:48:59 +02:00
}
if len ( t [ "extdirs" ] ) > 0 {
options = append ( options , "-extdirs" , t [ "extdirs" ] )
}
if len ( t [ "javaBuildDir" ] ) > 0 {
options = append ( options , "-java-build-dir" , t [ "javaBuildDir" ] )
}
if len ( t [ "source" ] ) > 0 {
options = append ( options , "-source" , t [ "source" ] )
}
if len ( t [ "jdk" ] ) > 0 {
options = append ( options , "-jdk" , t [ "jdk" ] )
}
if len ( t [ "sourcepath" ] ) > 0 {
options = append ( options , "-sourcepath" , t [ "sourcepath" ] )
}
2020-06-02 13:47:07 +02:00
case "pip" :
2020-05-25 19:48:59 +02:00
if len ( t [ "autoClasspath" ] ) > 0 {
options = append ( options , "-python-path" , t [ "autoClasspath" ] )
} else if len ( t [ "pythonPath" ] ) > 0 {
options = append ( options , "-python-path" , t [ "pythonPath" ] )
}
if len ( t [ "djangoTemplatDirs" ] ) > 0 {
options = append ( options , "-django-template-dirs" , t [ "djangoTemplatDirs" ] )
}
2020-06-02 13:47:07 +02:00
default :
return options
}
if len ( t [ "exclude" ] ) > 0 {
options = append ( options , "-exclude" , t [ "exclude" ] )
}
return append ( options , strings . Split ( t [ "src" ] , ":" ) ... )
}
func getSuppliedOrDefaultList ( suppliedList , defaultList [ ] string ) [ ] string {
if len ( suppliedList ) > 0 {
return suppliedList
}
return defaultList
}
func getSuppliedOrDefaultListAsString ( suppliedList , defaultList [ ] string , separator string ) string {
effectiveList := getSuppliedOrDefaultList ( suppliedList , defaultList )
return strings . Join ( effectiveList , separator )
}
// setTranslateEntryIfNotEmpty builds a string from either the user-supplied list, or the default list,
// by joining the entries with the given separator. If the resulting string is not empty, it will be
// placed as an entry in the provided map under the given key.
func setTranslateEntryIfNotEmpty ( translate map [ string ] interface { } , key , separator string , suppliedList , defaultList [ ] string ) {
value := getSuppliedOrDefaultListAsString ( suppliedList , defaultList , separator )
if value != "" {
translate [ key ] = value
}
}
// getSeparator returns the separator string depending on the host platform. This assumes that
// Piper executes the Fortify command line tools within the same OS platform as it is running on itself.
func getSeparator ( ) string {
if runtime . GOOS == "windows" {
return ";"
2020-05-25 19:48:59 +02:00
}
2020-06-02 13:47:07 +02:00
return ":"
2020-05-25 19:48:59 +02:00
}
2021-06-23 15:05:00 +02:00
2021-07-23 08:48:48 +02:00
func createToolRecordFortify ( workspace string , config fortifyExecuteScanOptions , projectID int64 , projectName string , projectVersionID int64 , projectVersion string ) ( string , error ) {
2021-06-23 15:05:00 +02:00
record := toolrecord . New ( workspace , "fortify" , config . ServerURL )
// Project
err := record . AddKeyData ( "project" ,
strconv . FormatInt ( projectID , 10 ) ,
projectName ,
"" )
if err != nil {
return "" , err
}
// projectVersion
2021-07-23 08:48:48 +02:00
projectVersionURL := config . ServerURL + "/html/ssc/version/" + strconv . FormatInt ( projectVersionID , 10 )
2021-06-23 15:05:00 +02:00
err = record . AddKeyData ( "projectVersion" ,
2021-07-23 08:48:48 +02:00
strconv . FormatInt ( projectVersionID , 10 ) ,
2021-06-23 15:05:00 +02:00
projectVersion ,
projectVersionURL )
if err != nil {
return "" , err
}
err = record . Persist ( )
if err != nil {
return "" , err
}
return record . GetFileName ( ) , nil
}