From 205bbc1bede0136cda2ad1f87cf086fd49bfe54c Mon Sep 17 00:00:00 2001 From: Giridhar Shenoy Date: Thu, 21 Jan 2021 14:57:00 +0100 Subject: [PATCH] fix(detect): scan paths to better handle signature scan and dependency scan paths (#2508) * changes to detectExec before master merge * changes for detectExecuteScan * self generated code added * fix syntax errors and update docu * added unit tests for fail and Group * fix failOn bug * add Groups as string array * add Groups as string array * tests and validation for groups, failOn * Updated docs and added more tests * documentation md files should not be changed * Handle merge conflicts from PR 1845 * fix merge errors * remove duplicate groups, merge error * adding buildCode and buildTool as params * switching build options * building maven modules * parameter correction * parameter correction * gnerate with new build parameter * adding comments * removing piper lib master and modifying goUtils to download 1.5.7 release * first cleaning then installing * multi module maven built * multi module maven built removing unwanted code * multi module maven built moving inside switch * testing * modifying the default use case to also call maven build * modifying the default use case to also call maven build wih -- * corrected maven build command * corrected maven build command with %v * skipping test runs * testing for MTA project with single pom * adding absolute path to m2 path * clean up * adding switch for mta and maven and removing env from containers * commiting changes for new detect step * correting log message * code clean up * unit tests changes to detectExecute * basic tests for new change * restoring piperGoUtils to download correct piper binary * code clean up * code clean up * revert to clean version * add unmap parameter to detect * Added Additional parameters for BlackDuck scan * Added detect tools paramater * fix detect.yaml to accept correct data type * fix codeclimate issue in detect.yaml Co-authored-by: Keshav Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> --- cmd/detectExecuteScan.go | 30 ++++++-- cmd/detectExecuteScan_generated.go | 90 ++++++++++++++++++++---- cmd/detectExecuteScan_test.go | 107 +++++++++++++++++++++++++++-- resources/metadata/detect.yaml | 62 +++++++++++++++++ 4 files changed, 265 insertions(+), 24 deletions(-) diff --git a/cmd/detectExecuteScan.go b/cmd/detectExecuteScan.go index 38cce4f09..a0eb1d614 100644 --- a/cmd/detectExecuteScan.go +++ b/cmd/detectExecuteScan.go @@ -10,8 +10,6 @@ import ( piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/maven" - sliceUtils "github.com/SAP/jenkins-library/pkg/piperutils" - "github.com/SAP/jenkins-library/pkg/command" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" @@ -155,12 +153,34 @@ func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectU } args = append(args, fmt.Sprintf("--detect.code.location.name=\\\"%v\\\"", codeLocation)) - if sliceUtils.ContainsString(config.Scanners, "signature") { + if len(config.ScanPaths) > 0 && len(config.ScanPaths[0]) > 0 { args = append(args, fmt.Sprintf("--detect.blackduck.signature.scanner.paths=%v", strings.Join(config.ScanPaths, ","))) } - if sliceUtils.ContainsString(config.Scanners, "source") { - args = append(args, fmt.Sprintf("--detect.source.path=%v", config.ScanPaths[0])) + if len(config.DependencyPath) > 0 { + args = append(args, fmt.Sprintf("--detect.source.path=%v", config.DependencyPath)) + } else { + args = append(args, fmt.Sprintf("--detect.source.path='.'")) + } + + if config.Unmap { + args = append(args, fmt.Sprintf("--detect.project.codelocation.unmap=true")) + } + + if len(config.IncludedPackageManagers) > 0 { + args = append(args, fmt.Sprintf("--detect.included.detector.types=%v", strings.ToUpper(strings.Join(config.IncludedPackageManagers, ",")))) + } + + if len(config.ExcludedPackageManagers) > 0 { + args = append(args, fmt.Sprintf("--detect.excluded.detector.types=%v", strings.ToUpper(strings.Join(config.ExcludedPackageManagers, ",")))) + } + + if len(config.MavenExcludedScopes) > 0 { + args = append(args, fmt.Sprintf("--detect.maven.excluded.scopes=%v", strings.ToLower(strings.Join(config.MavenExcludedScopes, ",")))) + } + + if len(config.DetectTools) > 0 { + args = append(args, fmt.Sprintf("--detect.tools=%v", strings.Join(config.DetectTools, ","))) } mavenArgs, err := maven.DownloadAndGetMavenParameters(config.GlobalSettingsFile, config.ProjectSettingsFile, utils) diff --git a/cmd/detectExecuteScan_generated.go b/cmd/detectExecuteScan_generated.go index d98371318..7eafe72a2 100644 --- a/cmd/detectExecuteScan_generated.go +++ b/cmd/detectExecuteScan_generated.go @@ -14,21 +14,27 @@ import ( ) type detectExecuteScanOptions struct { - Token string `json:"token,omitempty"` - CodeLocation string `json:"codeLocation,omitempty"` - ProjectName string `json:"projectName,omitempty"` - Scanners []string `json:"scanners,omitempty"` - ScanPaths []string `json:"scanPaths,omitempty"` - ScanProperties []string `json:"scanProperties,omitempty"` - ServerURL string `json:"serverUrl,omitempty"` - Groups []string `json:"groups,omitempty"` - FailOn []string `json:"failOn,omitempty"` - Version string `json:"version,omitempty"` - VersioningModel string `json:"versioningModel,omitempty"` - ProjectSettingsFile string `json:"projectSettingsFile,omitempty"` - GlobalSettingsFile string `json:"globalSettingsFile,omitempty"` - M2Path string `json:"m2Path,omitempty"` - InstallArtifacts bool `json:"installArtifacts,omitempty"` + Token string `json:"token,omitempty"` + CodeLocation string `json:"codeLocation,omitempty"` + ProjectName string `json:"projectName,omitempty"` + Scanners []string `json:"scanners,omitempty"` + ScanPaths []string `json:"scanPaths,omitempty"` + DependencyPath string `json:"dependencyPath,omitempty"` + Unmap bool `json:"unmap,omitempty"` + ScanProperties []string `json:"scanProperties,omitempty"` + ServerURL string `json:"serverUrl,omitempty"` + Groups []string `json:"groups,omitempty"` + FailOn []string `json:"failOn,omitempty"` + Version string `json:"version,omitempty"` + VersioningModel string `json:"versioningModel,omitempty"` + ProjectSettingsFile string `json:"projectSettingsFile,omitempty"` + GlobalSettingsFile string `json:"globalSettingsFile,omitempty"` + M2Path string `json:"m2Path,omitempty"` + InstallArtifacts bool `json:"installArtifacts,omitempty"` + IncludedPackageManagers []string `json:"includedPackageManagers,omitempty"` + ExcludedPackageManagers []string `json:"excludedPackageManagers,omitempty"` + MavenExcludedScopes []string `json:"mavenExcludedScopes,omitempty"` + DetectTools []string `json:"detectTools,omitempty"` } // DetectExecuteScanCommand Executes Synopsys Detect scan @@ -96,6 +102,8 @@ func addDetectExecuteScanFlags(cmd *cobra.Command, stepConfig *detectExecuteScan cmd.Flags().StringVar(&stepConfig.ProjectName, "projectName", os.Getenv("PIPER_projectName"), "Name of the Synopsis Detect (formerly BlackDuck) project.") cmd.Flags().StringSliceVar(&stepConfig.Scanners, "scanners", []string{`signature`}, "List of scanners to be used for Synopsis Detect (formerly BlackDuck) scan.") cmd.Flags().StringSliceVar(&stepConfig.ScanPaths, "scanPaths", []string{`.`}, "List of paths which should be scanned by the Synopsis Detect (formerly BlackDuck) scan.") + cmd.Flags().StringVar(&stepConfig.DependencyPath, "dependencyPath", `.`, "Absolute Path of the dependency management file of the project. This path represents the folder which contains the pom file, package.json etc. If the project contains multiple pom files, provide the path to the parent pom file or the base folder of the project") + cmd.Flags().BoolVar(&stepConfig.Unmap, "unmap", false, "Unmap flag will unmap all previous code locations and keep only the current scan results in the specified project version. Set this parameter to true, when the project version needs to store only the latest scan results.") cmd.Flags().StringSliceVar(&stepConfig.ScanProperties, "scanProperties", []string{`--blackduck.signature.scanner.memory=4096`, `--blackduck.timeout=6000`, `--blackduck.trust.cert=true`, `--detect.report.timeout=4800`, `--logging.level.com.synopsys.integration=DEBUG`, `--detect.maven.excluded.scopes=test`}, "Properties passed to the Synopsis Detect (formerly BlackDuck) scan. You can find details in the [Synopsis Detect documentation](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/622846/Using+Synopsys+Detect+Properties)") cmd.Flags().StringVar(&stepConfig.ServerURL, "serverUrl", os.Getenv("PIPER_serverUrl"), "Server URL to the Synopsis Detect (formerly BlackDuck) Server.") cmd.Flags().StringSliceVar(&stepConfig.Groups, "groups", []string{}, "Users groups to be assigned for the Project") @@ -106,6 +114,10 @@ func addDetectExecuteScanFlags(cmd *cobra.Command, stepConfig *detectExecuteScan cmd.Flags().StringVar(&stepConfig.GlobalSettingsFile, "globalSettingsFile", os.Getenv("PIPER_globalSettingsFile"), "Path or url to the mvn settings file that should be used as global settings file") cmd.Flags().StringVar(&stepConfig.M2Path, "m2Path", os.Getenv("PIPER_m2Path"), "Path to the location of the local repository that should be used.") cmd.Flags().BoolVar(&stepConfig.InstallArtifacts, "installArtifacts", false, "If enabled, it will install all artifacts to the local maven repository to make them available before running detect. This is required if any maven module has dependencies to other modules in the repository and they were not installed before.") + cmd.Flags().StringSliceVar(&stepConfig.IncludedPackageManagers, "includedPackageManagers", []string{}, "The package managers that need to be included for this scan. Providing the package manager names with this parameter will ensure that the build descriptor file of that package manager will be searched in the scan folder For the complete list of possible values for this parameter, please check https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detector-types-included-(Advanced)") + cmd.Flags().StringSliceVar(&stepConfig.ExcludedPackageManagers, "excludedPackageManagers", []string{}, "The package managers that need to be excluded for this scan. Providing the package manager names with this parameter will ensure that the build descriptor file of that package manager will be ignored in the scan folder For the complete list of possible values for this parameter, please check https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#%5BhardBreak%5DDetector-types-excluded-(Advanced)") + cmd.Flags().StringSliceVar(&stepConfig.MavenExcludedScopes, "mavenExcludedScopes", []string{}, "The maven scopes that need to be excluded from the scan. For example, setting the value 'test' will exclude all components which are defined with a test scope in maven") + cmd.Flags().StringSliceVar(&stepConfig.DetectTools, "detectTools", []string{}, "The type of BlackDuck scanners to include while running the BlackDuck scan. By default All scanners are included. For the complete list of possible values, Please refer https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detect-tools-included") cmd.MarkFlagRequired("token") cmd.MarkFlagRequired("projectName") @@ -174,6 +186,22 @@ func detectExecuteScanMetadata() config.StepData { Mandatory: false, Aliases: []config.Alias{{Name: "detect/scanPaths"}}, }, + { + Name: "dependencyPath", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{{Name: "detect/dependencyPath"}}, + }, + { + Name: "unmap", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{{Name: "detect/unmap"}}, + }, { Name: "scanProperties", ResourceRef: []config.ResourceReference{}, @@ -259,6 +287,38 @@ func detectExecuteScanMetadata() config.StepData { Mandatory: false, Aliases: []config.Alias{}, }, + { + Name: "includedPackageManagers", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{{Name: "detect/includedPackageManagers"}}, + }, + { + Name: "excludedPackageManagers", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{{Name: "detect/excludedPackageManagers"}}, + }, + { + Name: "mavenExcludedScopes", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{{Name: "detect/mavenExcludedScopes"}}, + }, + { + Name: "detectTools", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{{Name: "detect/detectTools"}}, + }, }, }, Containers: []config.Container{ diff --git a/cmd/detectExecuteScan_test.go b/cmd/detectExecuteScan_test.go index a8e9e36e9..06a0001d5 100644 --- a/cmd/detectExecuteScan_test.go +++ b/cmd/detectExecuteScan_test.go @@ -62,14 +62,14 @@ func TestRunDetect(t *testing.T) { assert.NoError(t, err) assert.Equal(t, ".", utilsMock.Dir, "Wrong execution directory used") assert.Equal(t, "/bin/bash", utilsMock.Shell[0], "Bash shell expected") - expectedScript := "./detect.sh --blackduck.url= --blackduck.api.token= --detect.project.name=\\\"\\\" --detect.project.version.name=\\\"\\\" --detect.code.location.name=\\\"\\\"" + expectedScript := "./detect.sh --blackduck.url= --blackduck.api.token= --detect.project.name=\\\"\\\" --detect.project.version.name=\\\"\\\" --detect.code.location.name=\\\"\\\" --detect.source.path='.'" assert.Equal(t, expectedScript, utilsMock.Calls[0]) }) t.Run("failure case", func(t *testing.T) { t.Parallel() utilsMock := newDetectTestUtilsBundle() - utilsMock.ShouldFailOnCommand = map[string]error{"./detect.sh --blackduck.url= --blackduck.api.token= --detect.project.name=\\\"\\\" --detect.project.version.name=\\\"\\\" --detect.code.location.name=\\\"\\\"": fmt.Errorf("Test Error")} + utilsMock.ShouldFailOnCommand = map[string]error{"./detect.sh --blackduck.url= --blackduck.api.token= --detect.project.name=\\\"\\\" --detect.project.version.name=\\\"\\\" --detect.code.location.name=\\\"\\\" --detect.source.path='.'": fmt.Errorf("Test Error")} utilsMock.AddFile("detect.sh", []byte("")) err := runDetect(detectExecuteScanOptions{}, utilsMock) assert.EqualError(t, err, "Test Error") @@ -127,6 +127,7 @@ func TestAddDetectArgs(t *testing.T) { "--detect.project.version.name=\\\"1.0\\\"", "--detect.code.location.name=\\\"testName/1.0\\\"", "--detect.blackduck.signature.scanner.paths=path1,path2", + "--detect.source.path='.'", }, }, { @@ -152,7 +153,8 @@ func TestAddDetectArgs(t *testing.T) { "--detect.project.user.groups=\\\"testGroup\\\"", "--detect.policy.check.fail.on.severities=BLOCKER,MAJOR", "--detect.code.location.name=\\\"testLocation\\\"", - "--detect.source.path=path1", + "--detect.blackduck.signature.scanner.paths=path1,path2", + "--detect.source.path='.'", }, }, { @@ -178,7 +180,104 @@ func TestAddDetectArgs(t *testing.T) { "--detect.project.user.groups=\\\"testGroup\\\",\\\"testGroup2\\\"", "--detect.policy.check.fail.on.severities=BLOCKER,MAJOR", "--detect.code.location.name=\\\"testLocation\\\"", - "--detect.source.path=path1", + "--detect.blackduck.signature.scanner.paths=path1,path2", + "--detect.source.path='.'", + }, + }, + { + args: []string{"--testProp1=1"}, + options: detectExecuteScanOptions{ + ServerURL: "https://server.url", + Token: "apiToken", + ProjectName: "testName", + CodeLocation: "testLocation", + FailOn: []string{"BLOCKER", "MAJOR"}, + Scanners: []string{"source"}, + ScanPaths: []string{"path1", "path2"}, + Groups: []string{"testGroup", "testGroup2"}, + Version: "1.0", + VersioningModel: "major-minor", + DependencyPath: "pathx", + }, + expected: []string{ + "--testProp1=1", + "--blackduck.url=https://server.url", + "--blackduck.api.token=apiToken", + "--detect.project.name=\\\"testName\\\"", + "--detect.project.version.name=\\\"1.0\\\"", + "--detect.project.user.groups=\\\"testGroup\\\",\\\"testGroup2\\\"", + "--detect.policy.check.fail.on.severities=BLOCKER,MAJOR", + "--detect.code.location.name=\\\"testLocation\\\"", + "--detect.blackduck.signature.scanner.paths=path1,path2", + "--detect.source.path=pathx", + }, + }, + { + args: []string{"--testProp1=1"}, + options: detectExecuteScanOptions{ + ServerURL: "https://server.url", + Token: "apiToken", + ProjectName: "testName", + CodeLocation: "testLocation", + FailOn: []string{"BLOCKER", "MAJOR"}, + Scanners: []string{"source"}, + ScanPaths: []string{"path1", "path2"}, + Groups: []string{"testGroup", "testGroup2"}, + Version: "1.0", + VersioningModel: "major-minor", + DependencyPath: "pathx", + Unmap: true, + }, + expected: []string{ + "--testProp1=1", + "--blackduck.url=https://server.url", + "--blackduck.api.token=apiToken", + "--detect.project.name=\\\"testName\\\"", + "--detect.project.version.name=\\\"1.0\\\"", + "--detect.project.user.groups=\\\"testGroup\\\",\\\"testGroup2\\\"", + "--detect.policy.check.fail.on.severities=BLOCKER,MAJOR", + "--detect.code.location.name=\\\"testLocation\\\"", + "--detect.blackduck.signature.scanner.paths=path1,path2", + "--detect.source.path=pathx", + "--detect.project.codelocation.unmap=true", + }, + }, + { + args: []string{"--testProp1=1"}, + options: detectExecuteScanOptions{ + ServerURL: "https://server.url", + Token: "apiToken", + ProjectName: "testName", + CodeLocation: "testLocation", + FailOn: []string{"BLOCKER", "MAJOR"}, + Scanners: []string{"source"}, + ScanPaths: []string{"path1", "path2"}, + Groups: []string{"testGroup", "testGroup2"}, + Version: "1.0", + VersioningModel: "major-minor", + DependencyPath: "pathx", + Unmap: true, + IncludedPackageManagers: []string{"maven", "GRADLE"}, + ExcludedPackageManagers: []string{"npm", "NUGET"}, + MavenExcludedScopes: []string{"TEST", "compile"}, + DetectTools: []string{"DETECTOR"}, + }, + expected: []string{ + "--testProp1=1", + "--blackduck.url=https://server.url", + "--blackduck.api.token=apiToken", + "--detect.project.name=\\\"testName\\\"", + "--detect.project.version.name=\\\"1.0\\\"", + "--detect.project.user.groups=\\\"testGroup\\\",\\\"testGroup2\\\"", + "--detect.policy.check.fail.on.severities=BLOCKER,MAJOR", + "--detect.code.location.name=\\\"testLocation\\\"", + "--detect.blackduck.signature.scanner.paths=path1,path2", + "--detect.source.path=pathx", + "--detect.project.codelocation.unmap=true", + "--detect.included.detector.types=MAVEN,GRADLE", + "--detect.excluded.detector.types=NPM,NUGET", + "--detect.maven.excluded.scopes=test,compile", + "--detect.tools=DETECTOR", }, }, } diff --git a/resources/metadata/detect.yaml b/resources/metadata/detect.yaml index 39b1217db..2a42fc07c 100644 --- a/resources/metadata/detect.yaml +++ b/resources/metadata/detect.yaml @@ -85,6 +85,26 @@ spec: - PARAMETERS - STAGES - STEPS + - name: dependencyPath + description: Absolute Path of the dependency management file of the project. This path represents the folder which contains the pom file, package.json etc. If the project contains multiple pom files, provide the path to the parent pom file or the base folder of the project + aliases: + - name: detect/dependencyPath + type: "string" + default: "." + scope: + - PARAMETERS + - STAGES + - STEPS + - name: unmap + description: Unmap flag will unmap all previous code locations and keep only the current scan results in the specified project version. Set this parameter to true, when the project version needs to store only the latest scan results. + aliases: + - name: detect/unmap + type: bool + default: false + scope: + - PARAMETERS + - STAGES + - STEPS - name: scanProperties description: Properties passed to the Synopsis Detect (formerly BlackDuck) scan. You can find details in the [Synopsis Detect documentation](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/622846/Using+Synopsys+Detect+Properties) aliases: @@ -217,6 +237,48 @@ spec: - STEPS - STAGES - PARAMETERS + - name: includedPackageManagers + description: + "The package managers that need to be included for this scan. Providing the package manager names with this parameter will ensure that the build descriptor file of that package manager will be searched in the scan folder + For the complete list of possible values for this parameter, please check https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detector-types-included-(Advanced)" + aliases: + - name: detect/includedPackageManagers + type: "[]string" + scope: + - PARAMETERS + - STAGES + - STEPS + - name: excludedPackageManagers + description: + "The package managers that need to be excluded for this scan. Providing the package manager names with this parameter will ensure that the build descriptor file of that package manager will be ignored in the scan folder + For the complete list of possible values for this parameter, please check https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#%5BhardBreak%5DDetector-types-excluded-(Advanced)" + aliases: + - name: detect/excludedPackageManagers + type: "[]string" + scope: + - PARAMETERS + - STAGES + - STEPS + - name: mavenExcludedScopes + description: "The maven scopes that need to be excluded from the scan. For example, setting the value 'test' will exclude all components which are defined with a test scope in maven" + aliases: + - name: detect/mavenExcludedScopes + type: "[]string" + scope: + - PARAMETERS + - STAGES + - STEPS + - name: detectTools + description: + "The type of BlackDuck scanners to include while running the BlackDuck scan. By default All scanners are included. + For the complete list of possible values, Please refer https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detect-tools-included" + aliases: + - name: detect/detectTools + type: "[]string" + scope: + - PARAMETERS + - STAGES + - STEPS containers: - name: openjdk image: openjdk:11