Feature - whitesourceExecuteScan - adding ability to scan multiple docker images (#4755)

* added-multiple-images-scan-logic

* amended-description

* added-reference-to-common-pipeline-env

cmd/whitesourceExecuteScan.go

@@ -196,28 +196,25 @@ func runWhitesourceExecuteScan(ctx context.Context, config *ScanOptions, scan *w
 }
 
 func runWhitesourceScan(ctx context.Context, config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment, influx *whitesourceExecuteScanInflux) error {
+
 	// Download Docker image for container scan
 	// ToDo: move it to improve testability
 	if config.BuildTool == "docker" {
-		saveImageOptions := containerSaveImageOptions{
+		if len(config.ScanImages) != 0 {
-			ContainerImage:            config.ScanImage,
+			for _, image := range config.ScanImages {
-			ContainerRegistryURL:      config.ScanImageRegistryURL,
+				config.ScanImage = image
-			ContainerRegistryUser:     config.ContainerRegistryUser,
+				err := downloadDockerImageAsTar(config, utils)
-			ContainerRegistryPassword: config.ContainerRegistryPassword,
+				if err != nil {
-			DockerConfigJSON:          config.DockerConfigJSON,
+					return errors.Wrapf(err, "failed to download docker image")
-			FilePath:                  config.ProjectName,
+				}
-			ImageFormat:               "legacy", // keep the image format legacy or whitesource is not able to read layers
-		}
-		dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", ImageFormat: "legacy"}
-		dClient := &piperDocker.Client{}
-		dClient.SetOptions(dClientOptions)
-		if _, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient, utils); err != nil {
-			if strings.Contains(fmt.Sprint(err), "no image found") {
-				log.SetErrorCategory(log.ErrorConfiguration)
 			}
-			return errors.Wrapf(err, "failed to download Docker image %v", config.ScanImage)
-		}
 
+		} else {
+			err := downloadDockerImageAsTar(config, utils)
+			if err != nil {
+				return errors.Wrapf(err, "failed to download docker image")
+			}
+		}
 	}
 
 	// Start the scan
@@ -375,8 +372,11 @@ func resolveProjectIdentifiers(config *ScanOptions, scan *ws.Scan, utils whiteso
 	if err := resolveProductToken(config, sys); err != nil {
 		return errors.Wrap(err, "error resolving product token")
 	}
-	if err := resolveAggregateProjectToken(config, sys); err != nil {
+
-		return errors.Wrap(err, "error resolving aggregate project token")
+	if !config.SkipParentProjectResolution {
+		if err := resolveAggregateProjectToken(config, sys); err != nil {
+			return errors.Wrap(err, "error resolving aggregate project token")
+		}
 	}
 
 	scan.ProductToken = config.ProductToken
@@ -465,33 +465,34 @@ func validateProductVersion(version string) string {
 
 func wsScanOptions(config *ScanOptions) *ws.ScanOptions {
 	return &ws.ScanOptions{
-		BuildTool:                  config.BuildTool,
+		BuildTool:                   config.BuildTool,
-		ScanType:                   "", // no longer provided via config
+		ScanType:                    "", // no longer provided via config
-		OrgToken:                   config.OrgToken,
+		OrgToken:                    config.OrgToken,
-		UserToken:                  config.UserToken,
+		UserToken:                   config.UserToken,
-		ProductName:                config.ProductName,
+		ProductName:                 config.ProductName,
-		ProductToken:               config.ProductToken,
+		ProductToken:                config.ProductToken,
-		ProductVersion:             config.Version,
+		ProductVersion:              config.Version,
-		ProjectName:                config.ProjectName,
+		ProjectName:                 config.ProjectName,
-		BuildDescriptorFile:        config.BuildDescriptorFile,
+		BuildDescriptorFile:         config.BuildDescriptorFile,
-		BuildDescriptorExcludeList: config.BuildDescriptorExcludeList,
+		BuildDescriptorExcludeList:  config.BuildDescriptorExcludeList,
-		PomPath:                    config.BuildDescriptorFile,
+		PomPath:                     config.BuildDescriptorFile,
-		M2Path:                     config.M2Path,
+		M2Path:                      config.M2Path,
-		GlobalSettingsFile:         config.GlobalSettingsFile,
+		GlobalSettingsFile:          config.GlobalSettingsFile,
-		ProjectSettingsFile:        config.ProjectSettingsFile,
+		ProjectSettingsFile:         config.ProjectSettingsFile,
-		InstallArtifacts:           config.InstallArtifacts,
+		InstallArtifacts:            config.InstallArtifacts,
-		DefaultNpmRegistry:         config.DefaultNpmRegistry,
+		DefaultNpmRegistry:          config.DefaultNpmRegistry,
-		AgentDownloadURL:           config.AgentDownloadURL,
+		AgentDownloadURL:            config.AgentDownloadURL,
-		AgentFileName:              config.AgentFileName,
+		AgentFileName:               config.AgentFileName,
-		ConfigFilePath:             config.ConfigFilePath,
+		ConfigFilePath:              config.ConfigFilePath,
-		Includes:                   config.Includes,
+		Includes:                    config.Includes,
-		Excludes:                   config.Excludes,
+		Excludes:                    config.Excludes,
-		JreDownloadURL:             config.JreDownloadURL,
+		JreDownloadURL:              config.JreDownloadURL,
-		AgentURL:                   config.AgentURL,
+		AgentURL:                    config.AgentURL,
-		ServiceURL:                 config.ServiceURL,
+		ServiceURL:                  config.ServiceURL,
-		ScanPath:                   config.ScanPath,
+		ScanPath:                    config.ScanPath,
-		InstallCommand:             config.InstallCommand,
+		InstallCommand:              config.InstallCommand,
-		Verbose:                    GeneralConfig.Verbose,
+		Verbose:                     GeneralConfig.Verbose,
+		SkipParentProjectResolution: config.SkipParentProjectResolution,
 	}
 }
 
@@ -1086,3 +1087,27 @@ func createToolRecordWhitesource(utils whitesourceUtils, workspace string, confi
 	}
 	return record.GetFileName(), nil
 }
+
+func downloadDockerImageAsTar(config *ScanOptions, utils whitesourceUtils) error {
+
+	saveImageOptions := containerSaveImageOptions{
+		ContainerImage:            config.ScanImage,
+		ContainerRegistryURL:      config.ScanImageRegistryURL,
+		ContainerRegistryUser:     config.ContainerRegistryUser,
+		ContainerRegistryPassword: config.ContainerRegistryPassword,
+		DockerConfigJSON:          config.DockerConfigJSON,
+		FilePath:                  config.ScanPath + "/" + config.ScanImage, // previously was config.ProjectName
+		ImageFormat:               "legacy",                                 // keep the image format legacy or whitesource is not able to read layers
+	}
+	dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", ImageFormat: "legacy"}
+	dClient := &piperDocker.Client{}
+	dClient.SetOptions(dClientOptions)
+	if _, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient, utils); err != nil {
+		if strings.Contains(fmt.Sprint(err), "no image found") {
+			log.SetErrorCategory(log.ErrorConfiguration)
+		}
+		return errors.Wrapf(err, "failed to download Docker image %v", config.ScanImage)
+	}
+
+	return nil
+}

cmd/whitesourceExecuteScan_generated.go

@@ -54,6 +54,8 @@ type whitesourceExecuteScanOptions struct {
 	ProjectToken                         string   `json:"projectToken,omitempty"`
 	Reporting                            bool     `json:"reporting,omitempty"`
 	ScanImage                            string   `json:"scanImage,omitempty"`
+	ScanImages                           []string `json:"scanImages,omitempty"`
+	SkipParentProjectResolution          bool     `json:"skipParentProjectResolution,omitempty"`
 	ScanImageRegistryURL                 string   `json:"scanImageRegistryUrl,omitempty"`
 	SecurityVulnerabilities              bool     `json:"securityVulnerabilities,omitempty"`
 	ServiceURL                           string   `json:"serviceUrl,omitempty"`
@@ -349,6 +351,8 @@ func addWhitesourceExecuteScanFlags(cmd *cobra.Command, stepConfig *whitesourceE
 	cmd.Flags().StringVar(&stepConfig.ProjectToken, "projectToken", os.Getenv("PIPER_projectToken"), "Project token to execute scan on. Ignored for scan types `maven`, `mta` and `npm`. Used for project aggregation when scanning with the Unified Agent and can be provided as an alternative to `projectName`.")
 	cmd.Flags().BoolVar(&stepConfig.Reporting, "reporting", true, "Whether assessment is being done at all, defaults to `true`")
 	cmd.Flags().StringVar(&stepConfig.ScanImage, "scanImage", os.Getenv("PIPER_scanImage"), "For `buildTool: docker`: Defines the docker image which should be scanned.")
+	cmd.Flags().StringSliceVar(&stepConfig.ScanImages, "scanImages", []string{}, "For `buildTool: docker`: Allowing to scan multiple docker images. In case parent project will not contain any dependecies, use skipParentProjectResolution parameter")
+	cmd.Flags().BoolVar(&stepConfig.SkipParentProjectResolution, "skipParentProjectResolution", false, "Parameter for multi-module, multi-images projects to skip the parent project resolution for reporing purpose Could be used if parent project is set as just a placeholder for scan and doesn't contain any dependencies.")
 	cmd.Flags().StringVar(&stepConfig.ScanImageRegistryURL, "scanImageRegistryUrl", os.Getenv("PIPER_scanImageRegistryUrl"), "For `buildTool: docker`: Defines the registry where the scanImage is located.")
 	cmd.Flags().BoolVar(&stepConfig.SecurityVulnerabilities, "securityVulnerabilities", true, "Whether security compliance is considered and reported as part of the assessment.")
 	cmd.Flags().StringVar(&stepConfig.ServiceURL, "serviceUrl", `https://saas.whitesourcesoftware.com/api`, "URL to the WhiteSource API endpoint.")
@@ -751,6 +755,29 @@ func whitesourceExecuteScanMetadata() config.StepData {
 						Aliases:   []config.Alias{},
 						Default:   os.Getenv("PIPER_scanImage"),
 					},
+					{
+						Name: "scanImages",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name:  "commonPipelineEnvironment",
+								Param: "container/imageNameTags",
+							},
+						},
+						Scope:     []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:      "[]string",
+						Mandatory: false,
+						Aliases:   []config.Alias{},
+						Default:   []string{},
+					},
+					{
+						Name:        "skipParentProjectResolution",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "bool",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     false,
+					},
 					{
 						Name: "scanImageRegistryUrl",
 						ResourceRef: []config.ResourceReference{

pkg/whitesource/configHelper.go

@@ -46,7 +46,14 @@ func (s *ScanOptions) RewriteUAConfigurationFile(utils Utils, projectName string
 	newConfig := properties.LoadMap(newConfigMap)
 
 	now := time.Now().Format("20060102150405")
-	newConfigFilePath := fmt.Sprintf("%v.%v", s.ConfigFilePath, now)
+
+	var newConfigFilePath string
+
+	if s.ScanPath != "." {
+		newConfigFilePath = fmt.Sprintf("%v/%v.%v", s.ScanPath, s.ConfigFilePath, now)
+	} else {
+		newConfigFilePath = fmt.Sprintf("%v.%v", s.ConfigFilePath, now)
+	}
 
 	var configContent bytes.Buffer
 	_, err = newConfig.Write(&configContent, properties.UTF8)

pkg/whitesource/scanOptions.go

@@ -46,5 +46,7 @@ type ScanOptions struct {
 
 	InstallCommand string
 
+	SkipParentProjectResolution bool
+
 	Verbose bool
 }

pkg/whitesource/scanUA.go

@@ -98,8 +98,10 @@ func (s *Scan) ExecuteUAScanInPath(config *ScanOptions, utils Utils, scanPath st
 	// ToDo: Check if Download of Docker/container image should be done here instead of in cmd/whitesourceExecuteScan.go
 
 	// ToDo: check if this is required
-	if err := s.AppendScannedProject(s.AggregateProjectName); err != nil {
+	if !config.SkipParentProjectResolution {
-		return err
+		if err := s.AppendScannedProject(s.AggregateProjectName); err != nil {
+			return err
+		}
 	}
 
 	configPath, err := config.RewriteUAConfigurationFile(utils, s.AggregateProjectName)

resources/metadata/whitesourceExecuteScan.yaml

@@ -377,6 +377,25 @@ spec:
           - PARAMETERS
           - STAGES
           - STEPS
+      - name: scanImages
+        type: "[]string"
+        description: "For `buildTool: docker`: Allowing to scan multiple docker images. In case parent project will not contain any dependecies, use skipParentProjectResolution parameter"
+        resourceRef:
+          - name: commonPipelineEnvironment
+            param: container/imageNameTags
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+      - name: skipParentProjectResolution
+        type: bool
+        description: "Parameter for multi-module, multi-images projects to skip the parent project resolution for reporing purpose
+        Could be used if parent project is set as just a placeholder for scan and doesn't contain any dependencies."
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        default: false
       - name: scanImageRegistryUrl
         type: string
         description: "For `buildTool: docker`: Defines the registry where the scanImage is located."