feat(detect): Scanonchanges : Parameter to reduce scan running time and load on BlackDuck server (#2538)

* changes to detectExec before master merge * changes for detectExecuteScan * self generated code added * fix syntax errors and update docu * added unit tests for fail and Group * fix failOn bug * add Groups as string array * add Groups as string array * tests and validation for groups, failOn * Updated docs and added more tests * documentation md files should not be changed * Handle merge conflicts from PR 1845 * fix merge errors * remove duplicate groups, merge error * adding buildCode and buildTool as params * switching build options * building maven modules * parameter correction * parameter correction * gnerate with new build parameter * adding comments * removing piper lib master and modifying goUtils to download 1.5.7 release * first cleaning then installing * multi module maven built * multi module maven built removing unwanted code * multi module maven built moving inside switch * testing * modifying the default use case to also call maven build * modifying the default use case to also call maven build wih -- * corrected maven build command * corrected maven build command with %v * skipping test runs * testing for MTA project with single pom * adding absolute path to m2 path * clean up * adding switch for mta and maven and removing env from containers * commiting changes for new detect step * correting log message * code clean up * unit tests changes to detectExecute * basic tests for new change * restoring piperGoUtils to download correct piper binary * code clean up * code clean up * clean detect code * add the the scanOnChanges parameter to detect * fix codeclimate issue * updated detect.yaml Co-authored-by: Keshav <anil.keshav@sap.com> Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2025-10-30 23:57:50 +02:00 · 2021-01-29 10:17:02 +01:00
parent 0ed5cce53a
commit d3f31acc9f
4 changed files with 79 additions and 8 deletions
--- a/cmd/detectExecuteScan.go
+++ b/cmd/detectExecuteScan.go
@@ -77,7 +77,7 @@ func detectExecuteScan(config detectExecuteScanOptions, _ *telemetry.CustomData)

 func runDetect(config detectExecuteScanOptions, utils detectUtils) error {
 	// detect execution details, see https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/88440888/Sample+Synopsys+Detect+Scan+Configuration+Scenarios+for+Black+Duck
-	err := utils.DownloadFile("https://detect.synopsys.com/detect.sh", "detect.sh", nil, nil)
+	err := getDetectScript(config, utils)
 	if err != nil {
 		return fmt.Errorf("failed to download 'detect.sh' script: %w", err)
 	}
@@ -118,6 +118,13 @@ func runDetect(config detectExecuteScanOptions, utils detectUtils) error {
 	return utils.RunShell("/bin/bash", script)
 }

+func getDetectScript(config detectExecuteScanOptions, utils detectUtils) error {
+	if config.ScanOnChanges {
+		return utils.DownloadFile("https://raw.githubusercontent.com/blackducksoftware/detect_rescan/master/detect_rescan.sh", "detect.sh", nil, nil)
+	}
+	return utils.DownloadFile("https://detect.synopsys.com/detect.sh", "detect.sh", nil, nil)
+}
+
 func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectUtils) ([]string, error) {

 	coordinates := struct {
@@ -128,6 +135,10 @@ func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectU

 	_, detectVersionName := versioning.DetermineProjectCoordinates("", config.VersioningModel, coordinates)

+	if config.ScanOnChanges {
+		args = append(args, "--report")
+	}
+
 	args = append(args, config.ScanProperties...)

 	args = append(args, fmt.Sprintf("--blackduck.url=%v", config.ServerURL))
--- a/cmd/detectExecuteScan_generated.go
+++ b/cmd/detectExecuteScan_generated.go
@@ -35,6 +35,7 @@ type detectExecuteScanOptions struct {
 	ExcludedPackageManagers []string `json:"excludedPackageManagers,omitempty"`
 	MavenExcludedScopes     []string `json:"mavenExcludedScopes,omitempty"`
 	DetectTools             []string `json:"detectTools,omitempty"`
+	ScanOnChanges           bool     `json:"scanOnChanges,omitempty"`
 }

 // DetectExecuteScanCommand Executes Synopsys Detect scan
@@ -114,10 +115,11 @@ func addDetectExecuteScanFlags(cmd *cobra.Command, stepConfig *detectExecuteScan
 	cmd.Flags().StringVar(&stepConfig.GlobalSettingsFile, "globalSettingsFile", os.Getenv("PIPER_globalSettingsFile"), "Path or url to the mvn settings file that should be used as global settings file")
 	cmd.Flags().StringVar(&stepConfig.M2Path, "m2Path", os.Getenv("PIPER_m2Path"), "Path to the location of the local repository that should be used.")
 	cmd.Flags().BoolVar(&stepConfig.InstallArtifacts, "installArtifacts", false, "If enabled, it will install all artifacts to the local maven repository to make them available before running detect. This is required if any maven module has dependencies to other modules in the repository and they were not installed before.")
-	cmd.Flags().StringSliceVar(&stepConfig.IncludedPackageManagers, "includedPackageManagers", []string{}, "The package managers that need to be included for this scan. Providing the package manager names with this parameter will ensure that the build descriptor file of that package manager will be searched in the scan folder For the complete list of possible values for this parameter, please check https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detector-types-included-(Advanced)")
-	cmd.Flags().StringSliceVar(&stepConfig.ExcludedPackageManagers, "excludedPackageManagers", []string{}, "The package managers that need to be excluded for this scan. Providing the package manager names with this parameter will ensure that the build descriptor file of that package manager will be ignored in the scan folder For the complete list of possible values for this parameter, please check https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#%5BhardBreak%5DDetector-types-excluded-(Advanced)")
+	cmd.Flags().StringSliceVar(&stepConfig.IncludedPackageManagers, "includedPackageManagers", []string{}, "The package managers that need to be included for this scan. Providing the package manager names with this parameter will ensure that the build descriptor file of that package manager will be searched in the scan folder For the complete list of possible values for this parameter, please refer [Synopsys detect documentation](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detector-types-included-(Advanced))")
+	cmd.Flags().StringSliceVar(&stepConfig.ExcludedPackageManagers, "excludedPackageManagers", []string{}, "The package managers that need to be excluded for this scan. Providing the package manager names with this parameter will ensure that the build descriptor file of that package manager will be ignored in the scan folder For the complete list of possible values for this parameter, please refer [Synopsys detect documentation](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#%5BhardBreak%5DDetector-types-excluded-(Advanced))")
 	cmd.Flags().StringSliceVar(&stepConfig.MavenExcludedScopes, "mavenExcludedScopes", []string{}, "The maven scopes that need to be excluded from the scan. For example, setting the value 'test' will exclude all components which are defined with a test scope in maven")
-	cmd.Flags().StringSliceVar(&stepConfig.DetectTools, "detectTools", []string{}, "The type of BlackDuck scanners to include while running the BlackDuck scan. By default All scanners are included. For the complete list of possible values, Please refer https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detect-tools-included")
+	cmd.Flags().StringSliceVar(&stepConfig.DetectTools, "detectTools", []string{}, "The type of BlackDuck scanners to include while running the BlackDuck scan. By default All scanners are included. For the complete list of possible values, Please refer [Synopsys detect documentation](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detect-tools-included)")
+	cmd.Flags().BoolVar(&stepConfig.ScanOnChanges, "scanOnChanges", false, "This flag determines if the scan is submitted to the server. If set to true, then the scan request is submitted to the server only when changes are detected in the Open Source Bill of Materials If the flag is set to false, then the scan request is submitted to server regardless of any changes. For more details please refer to the [documentation](https://github.com/blackducksoftware/detect_rescan/blob/master/README.md)")

 	cmd.MarkFlagRequired("token")
 	cmd.MarkFlagRequired("projectName")
@@ -319,6 +321,14 @@ func detectExecuteScanMetadata() config.StepData {
 						Mandatory:   false,
 						Aliases:     []config.Alias{{Name: "detect/detectTools"}},
 					},
+					{
+						Name:        "scanOnChanges",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "bool",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+					},
 				},
 			},
 			Containers: []config.Container{
--- a/cmd/detectExecuteScan_test.go
+++ b/cmd/detectExecuteScan_test.go
@@ -280,6 +280,46 @@ func TestAddDetectArgs(t *testing.T) {
 				"--detect.tools=DETECTOR",
 			},
 		},
+		{
+			args: []string{"--testProp1=1"},
+			options: detectExecuteScanOptions{
+				ServerURL:               "https://server.url",
+				Token:                   "apiToken",
+				ProjectName:             "testName",
+				CodeLocation:            "testLocation",
+				FailOn:                  []string{"BLOCKER", "MAJOR"},
+				Scanners:                []string{"source"},
+				ScanPaths:               []string{"path1", "path2"},
+				Groups:                  []string{"testGroup", "testGroup2"},
+				Version:                 "1.0",
+				VersioningModel:         "major-minor",
+				DependencyPath:          "pathx",
+				Unmap:                   true,
+				IncludedPackageManagers: []string{"maven", "GRADLE"},
+				ExcludedPackageManagers: []string{"npm", "NUGET"},
+				MavenExcludedScopes:     []string{"TEST", "compile"},
+				DetectTools:             []string{"DETECTOR"},
+				ScanOnChanges:           true,
+			},
+			expected: []string{
+				"--testProp1=1",
+				"--report",
+				"--blackduck.url=https://server.url",
+				"--blackduck.api.token=apiToken",
+				"--detect.project.name=\\\"testName\\\"",
+				"--detect.project.version.name=\\\"1.0\\\"",
+				"--detect.project.user.groups=\\\"testGroup\\\",\\\"testGroup2\\\"",
+				"--detect.policy.check.fail.on.severities=BLOCKER,MAJOR",
+				"--detect.code.location.name=\\\"testLocation\\\"",
+				"--detect.blackduck.signature.scanner.paths=path1,path2",
+				"--detect.source.path=pathx",
+				"--detect.project.codelocation.unmap=true",
+				"--detect.included.detector.types=MAVEN,GRADLE",
+				"--detect.excluded.detector.types=NPM,NUGET",
+				"--detect.maven.excluded.scopes=test,compile",
+				"--detect.tools=DETECTOR",
+			},
+		},
 	}

 	for k, v := range testData {
--- a/resources/metadata/detect.yaml
+++ b/resources/metadata/detect.yaml
@@ -144,7 +144,7 @@ spec:
        description: Mark the current build as fail based on the policy categories applied.
        longDescription: |
          A list of policies can be provided which will be applied after the scan is completed. These policies if violated will mark the build/scan result as failed.
-          The list of accepted valed can be found at https://blackducksoftware.github.io/synopsys-detect/latest/properties/configuration/project/#fail-on-policy-violation-severities
+          The list of accepted values can be found at [Synopsys detect documentation](https://blackducksoftware.github.io/synopsys-detect/latest/properties/configuration/project/#fail-on-policy-violation-severities)
        aliases:
          - name: detect/failOn
        type: "[]string"
@@ -240,7 +240,7 @@ spec:
      - name: includedPackageManagers
        description:
          "The package managers that need to be included for this scan. Providing the package manager names with this parameter will ensure that the build descriptor file of that package manager will be searched in the scan folder
-          For the complete list of possible values for this parameter, please check https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detector-types-included-(Advanced)"
+          For the complete list of possible values for this parameter, please refer [Synopsys detect documentation](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detector-types-included-(Advanced))"
        aliases:
          - name: detect/includedPackageManagers
        type: "[]string"
@@ -251,7 +251,7 @@ spec:
      - name: excludedPackageManagers
        description:
          "The package managers that need to be excluded for this scan. Providing the package manager names with this parameter will ensure that the build descriptor file of that package manager will be ignored in the scan folder
-          For the complete list of possible values for this parameter, please check https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#%5BhardBreak%5DDetector-types-excluded-(Advanced)"
+          For the complete list of possible values for this parameter, please refer [Synopsys detect documentation](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#%5BhardBreak%5DDetector-types-excluded-(Advanced))"
        aliases:
          - name: detect/excludedPackageManagers
        type: "[]string"
@@ -271,7 +271,7 @@ spec:
      - name: detectTools
        description:
          "The type of BlackDuck scanners to include while running the BlackDuck scan. By default All scanners are included.
-          For the complete list of possible values, Please refer https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detect-tools-included"
+          For the complete list of possible values, Please refer [Synopsys detect documentation](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631407160/Configuring+Detect+General+Properties#Detect-tools-included)"
        aliases:
          - name: detect/detectTools
        type: "[]string"
@@ -279,6 +279,16 @@ spec:
          - PARAMETERS
          - STAGES
          - STEPS
+      - name: scanOnChanges
+        description:
+          "This flag determines if the scan is submitted to the server. If set to true, then the scan request is submitted to the server only when changes are detected in the Open Source Bill of Materials
+          If the flag is set to false, then the scan request is submitted to server regardless of any changes.
+          For more details please refer to the [documentation](https://github.com/blackducksoftware/detect_rescan/blob/master/README.md)"
+        type: bool
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
  containers:
    - name: openjdk
      image: openjdk:11