feat(detectExecuteScan): Also scan images that are in the CPE (#4678)

feat(detectExecuteScan): Also scan images that are in the cpe

Signed-off-by: Ralf Pannemans <ralf.pannemans@sap.com>
Signed-off-by: Johannes Dillmann <j.dillmann@sap.com>
Signed-off-by: Pavel Busko <pavel.busko@sap.com>
Co-authored-by: Johannes Dillmann <j.dillmann@sap.com>
Co-authored-by: Pavel Busko <pavel.busko@sap.com>

cmd/detectExecuteScan.go

@@ -20,6 +20,7 @@ import (
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/maven"
 	"github.com/SAP/jenkins-library/pkg/orchestrator"
+	"github.com/SAP/jenkins-library/pkg/piperenv"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
 	"github.com/SAP/jenkins-library/pkg/reporting"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
@@ -30,6 +31,8 @@ import (
 	"github.com/pkg/errors"
 )
 
+const NO_VERSION_SUFFIX = ""
+
 type detectUtils interface {
 	piperutils.FileUtils
 
@@ -202,7 +205,7 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec
 	blackduckSystem := newBlackduckSystem(config)
 
 	args := []string{"./detect.sh"}
-	args, err = addDetectArgs(args, config, utils, blackduckSystem)
+	args, err = addDetectArgs(args, config, utils, blackduckSystem, NO_VERSION_SUFFIX, NO_VERSION_SUFFIX)
 	if err != nil {
 		return err
 	}
@@ -214,7 +217,18 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec
 	utils.SetDir(".")
 	utils.SetEnv(envs)
 
-	err = utils.RunShell("/bin/bash", script)
+	err = mapDetectError(utils.RunShell("/bin/bash", script), config, utils)
+	if config.ScanContainerDistro != "" {
+		imageError := mapDetectError(runDetectImages(ctx, config, utils, blackduckSystem, influx, blackduckSystem), config, utils)
+		if imageError != nil {
+			if err != nil {
+				err = errors.Wrapf(err, "error during scanning images: %q", imageError.Error())
+			} else {
+				err = imageError
+			}
+		}
+	}
+
 	reportingErr := postScanChecksAndReporting(ctx, config, influx, utils, blackduckSystem)
 	if reportingErr != nil {
 		if strings.Contains(reportingErr.Error(), "License Policy Violations found") {
@@ -227,6 +241,17 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec
 			log.Entry().Warnf("Failed to generate reports: %v", reportingErr)
 		}
 	}
+	// create Toolrecord file
+	toolRecordFileName, toolRecordErr := createToolRecordDetect(utils, "./", config, blackduckSystem)
+	if toolRecordErr != nil {
+		// do not fail until the framework is well established
+		log.Entry().Warning("TR_DETECT: Failed to create toolrecord file "+toolRecordFileName, err)
+	}
+
+	return err
+}
+
+func mapDetectError(err error, config detectExecuteScanOptions, utils detectUtils) error {
 	if err != nil {
 		// Setting error category based on exit code
 		mapErrorCategory(utils.GetExitCode())
@@ -238,15 +263,52 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec
 			err = errors.Wrapf(err, exitCodeMapping(utils.GetExitCode()))
 		}
 	}
-	// create Toolrecord file
-	toolRecordFileName, toolRecordErr := createToolRecordDetect(utils, "./", config, blackduckSystem)
-	if toolRecordErr != nil {
-		// do not fail until the framework is well established
-		log.Entry().Warning("TR_DETECT: Failed to create toolrecord file "+toolRecordFileName, err)
-	}
 	return err
 }
 
+func runDetectImages(ctx context.Context, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, influx *detectExecuteScanInflux, blackduckSystem *blackduckSystem) error {
+	cpePath := filepath.Join(GeneralConfig.EnvRootPath, "commonPipelineEnvironment")
+	imagesRaw := piperenv.GetResourceParameter(cpePath, "container", "imageNameTags.json")
+	if imagesRaw == "" {
+		log.Entry().Debugf("No images found to be scanned")
+		return nil
+	}
+
+	var err error
+	log.Entry().Infof("Scanning %d images", len(config.ImageNameTags))
+	for _, image := range config.ImageNameTags {
+		// Download image to be scanned
+		log.Entry().Debugf("Scanning image: %q", image)
+		tarName := fmt.Sprintf("%s.tar", strings.Split(image, ":")[0])
+
+		options := containerSaveImageOptions{
+			ContainerRegistryURL:      config.RegistryURL,
+			ContainerImage:            image,
+			ContainerRegistryPassword: config.RepositoryPassword,
+			ContainerRegistryUser:     config.RepositoryUsername,
+			FilePath:                  tarName,
+			ImageFormat:               "legacy",
+		}
+		containerSaveImage(options, &telemetry.CustomData{})
+
+		args := []string{"./detect.sh"}
+		args, err = addDetectArgsImages(args, config, utils, sys, tarName)
+		if err != nil {
+			return err
+		}
+		script := strings.Join(args, " ")
+
+		err = utils.RunShell("/bin/bash", script)
+		err = mapDetectError(err, config, utils)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // Get proper error category
 func mapErrorCategory(exitCodeKey int) {
 	switch exitCodeKey {
@@ -331,8 +393,11 @@ func getDetectScript(config detectExecuteScanOptions, utils detectUtils) error {
 	return nil
 }
 
-func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem) ([]string, error) {
+func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, versionSuffix, locationSuffix string) ([]string, error) {
 	detectVersionName := getVersionName(config)
+	if versionSuffix != NO_VERSION_SUFFIX {
+		detectVersionName = fmt.Sprintf("%s-%s", detectVersionName, versionSuffix)
+	}
 	// Split on spaces, the scanPropeties, so that each property is available as a single string
 	// instead of all properties being part of a single string
 	config.ScanProperties = piperutils.SplitAndTrim(config.ScanProperties, " ")
@@ -393,6 +458,10 @@ func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectU
 	codelocation := config.CodeLocation
 	if len(codelocation) == 0 && len(config.ProjectName) > 0 {
 		codelocation = fmt.Sprintf("%v/%v", config.ProjectName, detectVersionName)
+
+		if locationSuffix != "" {
+			codelocation = fmt.Sprintf("%v-%v", codelocation, locationSuffix)
+		}
 	}
 
 	args = append(args, fmt.Sprintf("\"--detect.project.name=%v\"", config.ProjectName))
@@ -467,6 +536,39 @@ func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectU
 	return args, nil
 }
 
+func addDetectArgsImages(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, imageTar string) ([]string, error) {
+	// suffix := strings.Split(imageTar, ".")[0]
+	// In order to preserve source scan result
+	config.Unmap = false
+	args, err := addDetectArgs(args, config, utils, sys, NO_VERSION_SUFFIX, fmt.Sprintf("image-%s", strings.Split(imageTar, ".")[0]))
+	if err != nil {
+		return []string{}, err
+	}
+
+	args = append(args, fmt.Sprintf("--detect.docker.tar=./%s", imageTar))
+	args = append(args, "--detect.target.type=IMAGE")
+	// https://community.synopsys.com/s/article/Docker-image-scanning-CLI-examples-and-some-Q-As
+	args = append(args, "--detect.tools.excluded=DETECTOR")
+	args = append(args, "--detect.docker.passthrough.shared.dir.path.local=/opt/blackduck/blackduck-imageinspector/shared/")
+	args = append(args, "--detect.docker.passthrough.shared.dir.path.imageinspector=/opt/blackduck/blackduck-imageinspector/shared")
+	args = append(args, fmt.Sprintf("--detect.docker.passthrough.imageinspector.service.distro.default=%s", config.ScanContainerDistro))
+	args = append(args, "--detect.docker.passthrough.imageinspector.service.start=false")
+	args = append(args, "--detect.docker.passthrough.output.include.squashedimage=false")
+
+	switch config.ScanContainerDistro {
+	case "ubuntu":
+		args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8082")
+	case "centos":
+		args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8081")
+	case "alpine":
+		args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8080")
+	default:
+		return nil, fmt.Errorf("unknown container distro %q", config.ScanContainerDistro)
+	}
+
+	return args, nil
+}
+
 func getVersionName(config detectExecuteScanOptions) string {
 	detectVersionName := config.CustomScanVersion
 	if len(detectVersionName) > 0 {

cmd/detectExecuteScan_generated.go

@@ -64,6 +64,11 @@ type detectExecuteScanOptions struct {
 	NpmArguments                []string `json:"npmArguments,omitempty"`
 	PrivateModules              string   `json:"privateModules,omitempty"`
 	PrivateModulesGitToken      string   `json:"privateModulesGitToken,omitempty"`
+	ScanContainerDistro         string   `json:"scanContainerDistro,omitempty" validate:"possible-values=ubuntu centos alpine"`
+	ImageNameTags               []string `json:"imageNameTags,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"`
+	RegistryURL                 string   `json:"registryUrl,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"`
+	RepositoryUsername          string   `json:"repositoryUsername,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"`
+	RepositoryPassword          string   `json:"repositoryPassword,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"`
 }
 
 type detectExecuteScanInflux struct {
@@ -310,6 +315,11 @@ func addDetectExecuteScanFlags(cmd *cobra.Command, stepConfig *detectExecuteScan
 	cmd.Flags().StringSliceVar(&stepConfig.NpmArguments, "npmArguments", []string{}, "List of additional arguments that Detect will add at then end of the npm ls command line when Detect executes the NPM CLI Detector on an NPM project.")
 	cmd.Flags().StringVar(&stepConfig.PrivateModules, "privateModules", os.Getenv("PIPER_privateModules"), "Tells go which modules shall be considered to be private (by setting [GOPRIVATE](https://pkg.go.dev/cmd/go#hdr-Configuration_for_downloading_non_public_code)).")
 	cmd.Flags().StringVar(&stepConfig.PrivateModulesGitToken, "privateModulesGitToken", os.Getenv("PIPER_privateModulesGitToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.")
+	cmd.Flags().StringVar(&stepConfig.ScanContainerDistro, "scanContainerDistro", os.Getenv("PIPER_scanContainerDistro"), "To also scan your images in the CPE, choose the distro")
+	cmd.Flags().StringSliceVar(&stepConfig.ImageNameTags, "imageNameTags", []string{}, "Images to be scanned (typically filled by CPE)")
+	cmd.Flags().StringVar(&stepConfig.RegistryURL, "registryUrl", os.Getenv("PIPER_registryUrl"), "Used accessing for the images to be scanned (typically filled by CPE)")
+	cmd.Flags().StringVar(&stepConfig.RepositoryUsername, "repositoryUsername", os.Getenv("PIPER_repositoryUsername"), "Used accessing for the images to be scanned (typically filled by CPE)")
+	cmd.Flags().StringVar(&stepConfig.RepositoryPassword, "repositoryPassword", os.Getenv("PIPER_repositoryPassword"), "Used accessing for the images to be scanned (typically filled by CPE)")
 
 	cmd.MarkFlagRequired("token")
 	cmd.MarkFlagRequired("projectName")
@@ -773,11 +783,81 @@ func detectExecuteScanMetadata() config.StepData {
 						Aliases:   []config.Alias{},
 						Default:   os.Getenv("PIPER_privateModulesGitToken"),
 					},
+					{
+						Name:        "scanContainerDistro",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_scanContainerDistro"),
+					},
+					{
+						Name: "imageNameTags",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name:  "commonPipelineEnvironment",
+								Param: "container/imageNameTags",
+							},
+						},
+						Scope:     []string{"STEPS", "STAGES", "PARAMETERS"},
+						Type:      "[]string",
+						Mandatory: false,
+						Aliases:   []config.Alias{},
+						Default:   []string{},
+					},
+					{
+						Name: "registryUrl",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name:  "commonPipelineEnvironment",
+								Param: "container/registryUrl",
+							},
+						},
+						Scope:     []string{"STEPS", "STAGES", "PARAMETERS"},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{},
+						Default:   os.Getenv("PIPER_registryUrl"),
+					},
+					{
+						Name: "repositoryUsername",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name:  "commonPipelineEnvironment",
+								Param: "container/repositoryUsername",
+							},
+						},
+						Scope:     []string{"STEPS", "STAGES", "PARAMETERS"},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{},
+						Default:   os.Getenv("PIPER_repositoryUsername"),
+					},
+					{
+						Name: "repositoryPassword",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name:  "commonPipelineEnvironment",
+								Param: "container/repositoryPassword",
+							},
+						},
+						Scope:     []string{"STEPS", "STAGES", "PARAMETERS"},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{},
+						Default:   os.Getenv("PIPER_repositoryPassword"),
+					},
 				},
 			},
 			Containers: []config.Container{
 				{Name: "openjdk", Image: "openjdk:11", WorkingDir: "/root", Options: []config.Option{{Name: "-u", Value: "0"}}},
 			},
+			Sidecars: []config.Container{
+				{Name: "inspector-ubuntu", Image: "blackducksoftware/blackduck-imageinspector-ubuntu:5.1.0", Conditions: []config.Condition{{ConditionRef: "strings-equal", Params: []config.Param{{Name: "scanContainerDistro", Value: "ubuntu"}}}}},
+				{Name: "inspector-alpine", Image: "blackducksoftware/blackduck-imageinspector-alpine:5.1.0", Conditions: []config.Condition{{ConditionRef: "strings-equal", Params: []config.Param{{Name: "scanContainerDistro", Value: "alpine"}}}}},
+				{Name: "inspector-centos", Image: "blackducksoftware/blackduck-imageinspector-centos:5.1.0", Conditions: []config.Condition{{ConditionRef: "strings-equal", Params: []config.Param{{Name: "scanContainerDistro", Value: "centos"}}}}},
+			},
 			Outputs: config.StepOutputs{
 				Resources: []config.StepResources{
 					{

cmd/detectExecuteScan_test.go

@@ -807,7 +807,7 @@ func TestAddDetectArgs(t *testing.T) {
 			config := detectExecuteScanOptions{Token: "token", ServerURL: "https://my.blackduck.system", ProjectName: v.options.ProjectName, Version: v.options.Version, CustomScanVersion: v.options.CustomScanVersion}
 			sys := newBlackduckMockSystem(config)
 
-			got, err := addDetectArgs(v.args, v.options, newDetectTestUtilsBundle(v.isPullRequest), &sys)
+			got, err := addDetectArgs(v.args, v.options, newDetectTestUtilsBundle(v.isPullRequest), &sys, NO_VERSION_SUFFIX, "")
 			assert.NoError(t, err)
 			assert.Equal(t, v.expected, got)
 		})

pkg/docker/docker.go

@@ -244,7 +244,8 @@ func (c *Client) DownloadImage(imageSource, targetFile string) (v1.Image, error)
 	craneCmd := cranecmd.NewCmdPull(&noOpts)
 	craneCmd.SetOut(log.Writer())
 	craneCmd.SetErr(log.Writer())
-	craneCmd.SetArgs([]string{imageRef.Name(), tmpFile.Name(), "--format=" + c.imageFormat})
+	args := []string{imageRef.Name(), tmpFile.Name(), "--format=" + c.imageFormat}
+	craneCmd.SetArgs(args)
 
 	if err := craneCmd.Execute(); err != nil {
 		defer os.Remove(tmpFile.Name())

resources/metadata/detectExecuteScan.yaml

@@ -511,6 +511,85 @@ spec:
           - type: vaultSecret
             name: golangPrivateModulesGitTokenVaultSecret
             default: golang
+      - name: scanContainerDistro
+        description: To also scan your images in the CPE, choose the distro
+        type: "string"
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        possibleValues:
+          - ubuntu
+          - centos
+          - alpine
+      - name: imageNameTags
+        type: "[]string"
+        mandatoryIf:
+          - name: scanContainerDistro
+            value: ubuntu
+          - name: scanContainerDistro
+            value: centos
+          - name: scanContainerDistro
+            value: alpine
+        description: Images to be scanned (typically filled by CPE)
+        scope:
+          - STEPS
+          - STAGES
+          - PARAMETERS
+        resourceRef:
+          - name: commonPipelineEnvironment
+            param: container/imageNameTags
+      - name: registryUrl
+        type: string
+        mandatoryIf:
+          - name: scanContainerDistro
+            value: ubuntu
+          - name: scanContainerDistro
+            value: centos
+          - name: scanContainerDistro
+            value: alpine
+        description: Used accessing for the images to be scanned (typically filled by CPE)
+        scope:
+          - STEPS
+          - STAGES
+          - PARAMETERS
+        resourceRef:
+          - name: commonPipelineEnvironment
+            param: container/registryUrl
+      - name: repositoryUsername
+        type: string
+        mandatoryIf:
+          - name: scanContainerDistro
+            value: ubuntu
+          - name: scanContainerDistro
+            value: centos
+          - name: scanContainerDistro
+            value: alpine
+        description: Used accessing for the images to be scanned (typically filled by CPE)
+        scope:
+          - STEPS
+          - STAGES
+          - PARAMETERS
+        resourceRef:
+          - name: commonPipelineEnvironment
+            param: container/repositoryUsername
+      - name: repositoryPassword
+        type: string
+        mandatoryIf:
+          - name: scanContainerDistro
+            value: ubuntu
+          - name: scanContainerDistro
+            value: centos
+          - name: scanContainerDistro
+            value: alpine
+        description: Used accessing for the images to be scanned (typically filled by CPE)
+        scope:
+          - STEPS
+          - STAGES
+          - PARAMETERS
+        resourceRef:
+          - name: commonPipelineEnvironment
+            param: container/repositoryPassword
   outputs:
     resources:
       - name: influx
@@ -562,3 +641,40 @@ spec:
       options:
         - name: -u
           value: "0"
+      volumeMounts:
+        - mountPath: /opt/blackduck/blackduck-imageinspector/shared
+          name: volume
+  sidecars:
+    - name: inspector-ubuntu
+      image: blackducksoftware/blackduck-imageinspector-ubuntu:5.1.0
+      command: ['']
+      volumeMounts:
+        - mountPath: /opt/blackduck/blackduck-imageinspector/shared
+          name: volume
+      conditions:
+        - conditionRef: strings-equal
+          params:
+            - name: scanContainerDistro
+              value: ubuntu
+    - name: inspector-alpine
+      image: blackducksoftware/blackduck-imageinspector-alpine:5.1.0
+      command: ['']
+      volumeMounts:
+        - mountPath: /opt/blackduck/blackduck-imageinspector/shared
+          name: volume
+      conditions:
+        - conditionRef: strings-equal
+          params:
+            - name: scanContainerDistro
+              value: alpine
+    - name: inspector-centos
+      image: blackducksoftware/blackduck-imageinspector-centos:5.1.0
+      command: ['']
+      volumeMounts:
+        - mountPath: /opt/blackduck/blackduck-imageinspector/shared
+          name: volume
+      conditions:
+        - conditionRef: strings-equal
+          params:
+            - name: scanContainerDistro
+              value: centos