diff --git a/cmd/containerSaveImage.go b/cmd/containerSaveImage.go index c1392ae91..ed4029632 100644 --- a/cmd/containerSaveImage.go +++ b/cmd/containerSaveImage.go @@ -19,21 +19,21 @@ func containerSaveImage(config containerSaveImageOptions, telemetryData *telemet dClient := &piperDocker.Client{} dClient.SetOptions(dClientOptions) - err := runContainerSaveImage(&config, telemetryData, cachePath, "", dClient) + _, err := runContainerSaveImage(&config, telemetryData, cachePath, "", dClient) if err != nil { log.Entry().WithError(err).Fatal("step execution failed") } } -func runContainerSaveImage(config *containerSaveImageOptions, telemetryData *telemetry.CustomData, cachePath, rootPath string, dClient piperDocker.Download) error { +func runContainerSaveImage(config *containerSaveImageOptions, telemetryData *telemetry.CustomData, cachePath, rootPath string, dClient piperDocker.Download) (string, error) { err := os.RemoveAll(cachePath) if err != nil { - return errors.Wrap(err, "failed to prepare cache") + return "", errors.Wrap(err, "failed to prepare cache") } err = os.Mkdir(cachePath, 0755) if err != nil { - return errors.Wrap(err, "failed to create cache") + return "", errors.Wrap(err, "failed to create cache") } // ensure that download cache is cleaned up at the end @@ -41,11 +41,11 @@ func runContainerSaveImage(config *containerSaveImageOptions, telemetryData *tel imageSource, err := dClient.GetImageSource() if err != nil { - return errors.Wrap(err, "failed to get docker image source") + return "", errors.Wrap(err, "failed to get docker image source") } image, err := dClient.DownloadImageToPath(imageSource, cachePath) if err != nil { - return errors.Wrap(err, "failed to download docker image") + return "", errors.Wrap(err, "failed to download docker image") } tarfilePath := config.FilePath @@ -57,20 +57,20 @@ func runContainerSaveImage(config *containerSaveImageOptions, telemetryData *tel tarFile, err := os.Create(tarfilePath) if err != nil { - return errors.Wrapf(err, "failed to create %v for docker image", tarfilePath) + return "", errors.Wrapf(err, "failed to create %v for docker image", tarfilePath) } defer tarFile.Close() if err := os.Chmod(tarfilePath, 0644); err != nil { - return errors.Wrapf(err, "failed to adapt permissions on %v", tarfilePath) + return "", errors.Wrapf(err, "failed to adapt permissions on %v", tarfilePath) } err = dClient.TarImage(tarFile, image) if err != nil { - return errors.Wrap(err, "failed to tar container image") + return "", errors.Wrap(err, "failed to tar container image") } - return nil + return tarfilePath, nil } func filenameFromContainer(rootPath, containerImage string) string { diff --git a/cmd/containerSaveImage_test.go b/cmd/containerSaveImage_test.go index a93e08692..b370ee8c7 100644 --- a/cmd/containerSaveImage_test.go +++ b/cmd/containerSaveImage_test.go @@ -65,7 +65,7 @@ func TestRunContainerSaveImage(t *testing.T) { dClient := containerMock{} - err = runContainerSaveImage(&config, &telemetryData, cacheFolder, tmpFolder, &dClient) + filePath, err := runContainerSaveImage(&config, &telemetryData, cacheFolder, tmpFolder, &dClient) assert.NoError(t, err) assert.Equal(t, cacheFolder, dClient.filePath) @@ -74,12 +74,14 @@ func TestRunContainerSaveImage(t *testing.T) { content, err := ioutil.ReadFile(filepath.Join(tmpFolder, "testfile.tar")) assert.NoError(t, err) assert.Equal(t, "This is a test", string(content)) + + assert.Contains(t, filePath, "testfile.tar") }) t.Run("failure - cache creation", func(t *testing.T) { config := containerSaveImageOptions{} dClient := containerMock{} - err := runContainerSaveImage(&config, &telemetryData, "", "", &dClient) + _, err := runContainerSaveImage(&config, &telemetryData, "", "", &dClient) assert.Contains(t, fmt.Sprint(err), "failed to create cache: mkdir :") }) @@ -92,7 +94,7 @@ func TestRunContainerSaveImage(t *testing.T) { defer os.RemoveAll(tmpFolder) dClient := containerMock{imageSourceErr: "image source error"} - err = runContainerSaveImage(&config, &telemetryData, filepath.Join(tmpFolder, "cache"), tmpFolder, &dClient) + _, err = runContainerSaveImage(&config, &telemetryData, filepath.Join(tmpFolder, "cache"), tmpFolder, &dClient) assert.EqualError(t, err, "failed to get docker image source: image source error") }) @@ -105,7 +107,7 @@ func TestRunContainerSaveImage(t *testing.T) { defer os.RemoveAll(tmpFolder) dClient := containerMock{downloadImageErr: "download error"} - err = runContainerSaveImage(&config, &telemetryData, filepath.Join(tmpFolder, "cache"), tmpFolder, &dClient) + _, err = runContainerSaveImage(&config, &telemetryData, filepath.Join(tmpFolder, "cache"), tmpFolder, &dClient) assert.EqualError(t, err, "failed to download docker image: download error") }) @@ -118,7 +120,7 @@ func TestRunContainerSaveImage(t *testing.T) { defer os.RemoveAll(tmpFolder) dClient := containerMock{tarImageErr: "tar error"} - err = runContainerSaveImage(&config, &telemetryData, filepath.Join(tmpFolder, "cache"), tmpFolder, &dClient) + _, err = runContainerSaveImage(&config, &telemetryData, filepath.Join(tmpFolder, "cache"), tmpFolder, &dClient) assert.EqualError(t, err, "failed to tar container image: tar error") }) } diff --git a/cmd/malwareExecuteScan.go b/cmd/malwareExecuteScan.go index b2f5e9029..27e734ce4 100644 --- a/cmd/malwareExecuteScan.go +++ b/cmd/malwareExecuteScan.go @@ -1,171 +1,177 @@ package cmd import ( - "crypto/sha256" "encoding/json" "fmt" - "github.com/SAP/jenkins-library/pkg/command" + piperDocker "github.com/SAP/jenkins-library/pkg/docker" piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/malwarescan" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/telemetry" + "github.com/SAP/jenkins-library/pkg/toolrecord" "github.com/pkg/errors" "io" - "io/ioutil" - "net/http" "os" + "path/filepath" + "strings" "time" ) -var open = _open -var getSHA256 = _getSHA256 +type malwareScanUtils interface { + OpenFile(name string, flag int, perm os.FileMode) (io.ReadCloser, error) + SHA256(path string) (string, error) -func _open(path string) (io.ReadCloser, error) { - return os.Open(path) + newDockerClient(piperDocker.ClientOptions) piperDocker.Download + + malwarescan.Client + piperutils.FileUtils } -type malwareExecuteScanResponse struct { - MalwareDetected bool - EncryptedContentDetected bool - ScanSize int - MimeType string - SHA256 string +type malwareScanUtilsBundle struct { + malwarescan.Client + *piperutils.Files } -func malwareExecuteScan(config malwareExecuteScanOptions, telemetryData *telemetry.CustomData) { - // for command execution use Command - c := command.Command{} - // reroute command output to logging framework - c.Stdout(log.Writer()) - c.Stderr(log.Writer()) - - // for http calls import piperhttp "github.com/SAP/jenkins-library/pkg/http" - // and use a &piperhttp.Client{} in a custom system - // Example: step checkmarxExecuteScan.go - - httpClient := &piperhttp.Client{} - - // error situations should stop execution through log.Entry().Fatal() call which leads to an os.Exit(1) in the end - err := runMalwareScan(&config, telemetryData, &c, httpClient) - if err != nil { - log.Entry().WithError(err).Fatal("step execution failed") - } +func (utils *malwareScanUtilsBundle) OpenFile(name string, flag int, perm os.FileMode) (io.ReadCloser, error) { + return utils.Files.FileOpen(name, flag, perm) } -func runMalwareScan(config *malwareExecuteScanOptions, telemetryData *telemetry.CustomData, command command.ExecRunner, - httpClient piperhttp.Sender) error { - - log.Entry().Infof("Scanning file \"%s\" for malware using service \"%s\"", config.File, config.Host) - - candidate, err := open(config.File) - if err != nil { - return err - } - defer candidate.Close() +func (utils *malwareScanUtilsBundle) newDockerClient(options piperDocker.ClientOptions) piperDocker.Download { + dClient := piperDocker.Client{} + dClient.SetOptions(options) + return &dClient +} +func newMalwareScanUtilsBundle(config malwareExecuteScanOptions) *malwareScanUtilsBundle { timeout, err := time.ParseDuration(fmt.Sprintf("%ss", config.Timeout)) if err != nil { - return errors.Wrapf(err, "Invalid timeout: %v", config.Timeout) + timeout = 60 + log.Entry().Warnf("Unable to parse timeout for malwareScan: '%v'. Falling back to %ds", err, timeout) } - opts := piperhttp.ClientOptions{ + httpClientOptions := piperhttp.ClientOptions{ Username: config.Username, Password: config.Password, MaxRequestDuration: timeout, TransportTimeout: timeout, } - httpClient.SetOptions(opts) - var scanResponse *malwareExecuteScanResponse - scanResponse, err = sendMalwareScanRequest(httpClient, "POST", config.Host+"/scan", candidate) + httpClient := &piperhttp.Client{} + httpClient.SetOptions(httpClientOptions) + + return &malwareScanUtilsBundle{ + Client: &malwarescan.ClientImpl{ + HTTPClient: httpClient, + Host: config.Host, + }, + Files: &piperutils.Files{}, + } +} + +func malwareExecuteScan(config malwareExecuteScanOptions, telemetryData *telemetry.CustomData) { + utils := newMalwareScanUtilsBundle(config) + + err := runMalwareScan(&config, telemetryData, utils) + if err != nil { + log.Entry().WithError(err).Fatal("step execution failed") + } +} + +func runMalwareScan(config *malwareExecuteScanOptions, telemetryData *telemetry.CustomData, utils malwareScanUtils) error { + file, err := selectAndPrepareFileForMalwareScan(config, utils) + if err != nil { + return err + } + + log.Entry().Infof("Scanning file \"%s\" for malware using service \"%s\"", file, config.Host) + + candidate, err := utils.OpenFile(file, os.O_RDONLY, 0666) + if err != nil { + return err + } + defer candidate.Close() + + scannerInfo, err := utils.Info() + + log.Entry().Infof("***************************************") + log.Entry().Infof("* Engine: %s", scannerInfo.EngineVersion) + log.Entry().Infof("* Signatures: %s", scannerInfo.SignatureTimestamp) + log.Entry().Infof("***************************************") + + if _, err = createToolRecordMalwareScan("./", config, scannerInfo); err != nil { + return err + } + + scanResponse, err := utils.Scan(candidate) if err != nil { return err } + if err = createMalwareScanReport(config, scanResponse, utils); err != nil { + return err + } + log.Entry().Debugf( - "File '%s' has been scanned. MalwareDetected: %t, EncryptedContentDetected: %t, ScanSize: %d, MimeType: '%s', SHA256: '%s'", - config.File, + "File '%s' has been scanned. MalwareDetected: %t, EncryptedContentDetected: %t, ScanSize: %d, MimeType: '%s', SHA256: '%s', Finding: '%s'", + file, scanResponse.MalwareDetected, scanResponse.EncryptedContentDetected, scanResponse.ScanSize, scanResponse.MimeType, - scanResponse.SHA256) + scanResponse.SHA256, + scanResponse.Finding) - err = validateHash(scanResponse.SHA256, config.File) - if err != nil { + if err = validateHash(scanResponse.SHA256, file, utils); err != nil { return err } if scanResponse.MalwareDetected || scanResponse.EncryptedContentDetected { - return fmt.Errorf("Malware scan failed for file '%s'. Malware detected: %t, encrypted content detected: %t", - config.File, scanResponse.MalwareDetected, scanResponse.EncryptedContentDetected) + return fmt.Errorf("Malware scan failed for file '%s'. Malware detected: %t, encrypted content detected: %t, finding: %v", + file, scanResponse.MalwareDetected, scanResponse.EncryptedContentDetected, scanResponse.Finding) } log.Entry().Infof("Malware scan succeeded for file '%s'. Malware detected: %t, encrypted content detected: %t", - config.File, scanResponse.MalwareDetected, scanResponse.EncryptedContentDetected) + file, scanResponse.MalwareDetected, scanResponse.EncryptedContentDetected) return nil } -func sendMalwareScanRequest(client piperhttp.Sender, method, url string, candidate io.Reader) (*malwareExecuteScanResponse, error) { - - // piper http utils mashall some http response codes into errors. We wan't to check the status code - // ourselves hence we wait with returning that error (maybe also related to errors others than http status codes) - - // sendRequest results in any combination of nil and non-nil response and error. - // a response body could even be already closed. - response, err := client.SendRequest(method, url, candidate, prepareHeaders(), nil) - - if response != nil && response.Body != nil { - defer response.Body.Close() +func selectAndPrepareFileForMalwareScan(config *malwareExecuteScanOptions, utils malwareScanUtils) (string, error) { + if len(config.ScanFile) > 0 { + return config.ScanFile, nil } - return validateResponse(response, err) + // automatically detect the file to be scanned depending on the buildtool + if config.BuildTool == "docker" && len(config.ScanImage) > 0 { + correctMalwareDockerConfigEnvVar(config, utils) + + saveImageOptions := containerSaveImageOptions{ + ContainerImage: config.ScanImage, + ContainerRegistryURL: config.ScanImageRegistryURL, + IncludeLayers: config.ScanImageIncludeLayers, + FilePath: config.ScanImage, + } + dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", IncludeLayers: saveImageOptions.IncludeLayers} + dClient := utils.newDockerClient(dClientOptions) + + tarFile, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient) + + if err != nil { + if strings.Contains(fmt.Sprint(err), "no image found") { + log.SetErrorCategory(log.ErrorConfiguration) + } + return "", errors.Wrapf(err, "failed to download Docker image %v", config.ScanImage) + } + return tarFile, nil + } + + return "", fmt.Errorf("Please specify a file to be scanned") } -func validateResponse(response *http.Response, err error) (*malwareExecuteScanResponse, error) { - - var body []byte - var errRead error - if response != nil && response.Body != nil { - body, errRead = ioutil.ReadAll(response.Body) - } - - if err != nil { - return nil, fmt.Errorf("HTTP request failed with error: %v. Details: \"%s\"", err, body) - } - - if response == nil { - return nil, fmt.Errorf("No response available") - } - - if response.StatusCode != 200 { - return nil, fmt.Errorf("Unexpected response code (%d). %d expected. Details: \"%s\"", response.StatusCode, 200, body) - } - - if errRead != nil { - return nil, errRead - } - - return marshalResponse(body) -} - -func marshalResponse(body []byte) (*malwareExecuteScanResponse, error) { - - var scanResponse malwareExecuteScanResponse - - err := json.Unmarshal(body, &scanResponse) - - if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("Unmarshalling of response body failed. Body: '%s'", body)) - } - - return &scanResponse, nil -} - -func validateHash(remoteHash, fileName string) error { - - hash, err := getSHA256(fileName) +func validateHash(remoteHash, fileName string, utils malwareScanUtils) error { + hash, err := utils.SHA256(fileName) if err != nil { return err } @@ -180,25 +186,52 @@ func validateHash(remoteHash, fileName string) error { return nil } -func _getSHA256(fileName string) (string, error) { +// create toolrecord file for malwarescan +func createToolRecordMalwareScan(workspace string, config *malwareExecuteScanOptions, scanner *malwarescan.Info) (string, error) { + record := toolrecord.New(workspace, "malwarescan", config.Host) + record.SetOverallDisplayData("Malware Scanner", "") - f, err := open(fileName) - if err != nil { - return "", err - } - defer f.Close() - - hash := sha256.New() - _, err = io.Copy(hash, f) - if err != nil { + if err := record.AddKeyData("engineVersion", scanner.EngineVersion, "Engine Version", ""); err != nil { return "", err } - return fmt.Sprintf("%x", string(hash.Sum(nil))), nil + if err := record.AddKeyData("signatureTimestamp", scanner.SignatureTimestamp, "Signature Timestamp", ""); err != nil { + return "", err + } + + if err := record.Persist(); err != nil { + return "", err + } + + return record.GetFileName(), nil } -func prepareHeaders() http.Header { - headers := http.Header{} - headers.Add("Content-Type", "application/octet-stream") - return headers +func createMalwareScanReport(config *malwareExecuteScanOptions, scanResult *malwarescan.ScanResult, utils malwareScanUtils) error { + scanResultJSON, err := json.Marshal(scanResult) + + if err != nil { + return err + } + + return utils.FileWrite(config.ReportFileName, scanResultJSON, 0666) +} + +func correctMalwareDockerConfigEnvVar(config *malwareExecuteScanOptions, utils malwareScanUtils) { + path := config.DockerConfigJSON + if len(path) > 0 { + log.Entry().Infof("Docker credentials configuration: %v", path) + if len(config.ScanImageRegistryURL) > 0 && len(config.ContainerRegistryUser) > 0 && len(config.ContainerRegistryPassword) > 0 { + var err error + path, err = piperDocker.CreateDockerConfigJSON(config.ScanImageRegistryURL, config.ContainerRegistryUser, config.ContainerRegistryPassword, "", config.DockerConfigJSON, utils) + if err != nil { + log.Entry().Warningf("failed to update Docker config.json: %v", err) + } + } + path, _ := utils.Abs(path) + // use parent directory + path = filepath.Dir(path) + os.Setenv("DOCKER_CONFIG", path) + } else { + log.Entry().Info("Docker credentials configuration: NONE") + } } diff --git a/cmd/malwareExecuteScan_generated.go b/cmd/malwareExecuteScan_generated.go index 76cc902fe..52aaa0c7a 100644 --- a/cmd/malwareExecuteScan_generated.go +++ b/cmd/malwareExecuteScan_generated.go @@ -5,39 +5,88 @@ package cmd import ( "fmt" "os" + "reflect" + "strings" "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcs" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" "github.com/SAP/jenkins-library/pkg/validation" + "github.com/bmatcuk/doublestar" "github.com/spf13/cobra" ) type malwareExecuteScanOptions struct { - Host string `json:"host,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - File string `json:"file,omitempty"` - Timeout string `json:"timeout,omitempty"` + BuildTool string `json:"buildTool,omitempty"` + DockerConfigJSON string `json:"dockerConfigJSON,omitempty"` + ContainerRegistryPassword string `json:"containerRegistryPassword,omitempty"` + ContainerRegistryUser string `json:"containerRegistryUser,omitempty"` + Host string `json:"host,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + ScanImage string `json:"scanImage,omitempty"` + ScanImageIncludeLayers bool `json:"scanImageIncludeLayers,omitempty"` + ScanImageRegistryURL string `json:"scanImageRegistryUrl,omitempty"` + ScanFile string `json:"scanFile,omitempty"` + Timeout string `json:"timeout,omitempty"` + ReportFileName string `json:"reportFileName,omitempty"` } -// MalwareExecuteScanCommand Performs a malware scan +type malwareExecuteScanReports struct { +} + +func (p *malwareExecuteScanReports) persist(stepConfig malwareExecuteScanOptions, gcpJsonKeyFilePath string, gcsBucketId string, gcsFolderPath string, gcsSubFolder string) { + if gcsBucketId == "" { + log.Entry().Info("persisting reports to GCS is disabled, because gcsBucketId is empty") + return + } + log.Entry().Info("Uploading reports to Google Cloud Storage...") + content := []gcs.ReportOutputParam{ + {FilePattern: "**/toolrun_malwarescan_*.json", ParamRef: "", StepResultType: "malwarescan"}, + {FilePattern: "", ParamRef: "reportFileName", StepResultType: "malwarescan"}, + } + envVars := []gcs.EnvVar{ + {Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: gcpJsonKeyFilePath, Modified: false}, + } + gcsClient, err := gcs.NewClient(gcs.WithEnvVars(envVars)) + if err != nil { + log.Entry().Errorf("creation of GCS client failed: %v", err) + } + defer gcsClient.Close() + structVal := reflect.ValueOf(&stepConfig).Elem() + inputParameters := map[string]string{} + for i := 0; i < structVal.NumField(); i++ { + field := structVal.Type().Field(i) + if field.Type.String() == "string" { + paramName := strings.Split(field.Tag.Get("json"), ",") + paramValue, _ := structVal.Field(i).Interface().(string) + inputParameters[paramName[0]] = paramValue + } + } + if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, gcsFolderPath, gcsBucketId, gcsSubFolder, doublestar.Glob, os.Stat); err != nil { + log.Entry().Errorf("failed to persist reports: %v", err) + } +} + +// MalwareExecuteScanCommand Performs a malware scan using the [SAP Malware Scanning Service](https://help.sap.com/viewer/b416237f818c4e2e827f6118640079f8/LATEST/en-US/b7c9b86fe724458086a502df3160f380.html). func MalwareExecuteScanCommand() *cobra.Command { const STEP_NAME = "malwareExecuteScan" metadata := malwareExecuteScanMetadata() var stepConfig malwareExecuteScanOptions var startTime time.Time + var reports malwareExecuteScanReports var logCollector *log.CollectorHook var splunkClient *splunk.Splunk telemetryClient := &telemetry.Telemetry{} var createMalwareExecuteScanCmd = &cobra.Command{ Use: STEP_NAME, - Short: "Performs a malware scan", - Long: `Performs a malware scan`, + Short: "Performs a malware scan using the [SAP Malware Scanning Service](https://help.sap.com/viewer/b416237f818c4e2e827f6118640079f8/LATEST/en-US/b7c9b86fe724458086a502df3160f380.html).", + Long: `Performs a malware scan using the [SAP Malware Scanning Service](https://help.sap.com/viewer/b416237f818c4e2e827f6118640079f8/LATEST/en-US/b7c9b86fe724458086a502df3160f380.html).`, PreRunE: func(cmd *cobra.Command, _ []string) error { startTime = time.Now() log.SetStepName(STEP_NAME) @@ -54,6 +103,9 @@ func MalwareExecuteScanCommand() *cobra.Command { log.SetErrorCategory(log.ErrorConfiguration) return err } + log.RegisterSecret(stepConfig.DockerConfigJSON) + log.RegisterSecret(stepConfig.ContainerRegistryPassword) + log.RegisterSecret(stepConfig.ContainerRegistryUser) log.RegisterSecret(stepConfig.Username) log.RegisterSecret(stepConfig.Password) @@ -83,6 +135,7 @@ func MalwareExecuteScanCommand() *cobra.Command { stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { + reports.persist(stepConfig, GeneralConfig.GCPJsonKeyFilePath, GeneralConfig.GCSBucketId, GeneralConfig.GCSFolderPath, GeneralConfig.GCSSubFolder) config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() @@ -114,16 +167,24 @@ func MalwareExecuteScanCommand() *cobra.Command { } func addMalwareExecuteScanFlags(cmd *cobra.Command, stepConfig *malwareExecuteScanOptions) { + cmd.Flags().StringVar(&stepConfig.BuildTool, "buildTool", os.Getenv("PIPER_buildTool"), "Defines the tool which is used for building the artifact.") + cmd.Flags().StringVar(&stepConfig.DockerConfigJSON, "dockerConfigJSON", os.Getenv("PIPER_dockerConfigJSON"), "Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).") + cmd.Flags().StringVar(&stepConfig.ContainerRegistryPassword, "containerRegistryPassword", os.Getenv("PIPER_containerRegistryPassword"), "For `buildTool: docker`: Password for container registry access - typically provided by the CI/CD environment.") + cmd.Flags().StringVar(&stepConfig.ContainerRegistryUser, "containerRegistryUser", os.Getenv("PIPER_containerRegistryUser"), "For `buildTool: docker`: Username for container registry access - typically provided by the CI/CD environment.") cmd.Flags().StringVar(&stepConfig.Host, "host", os.Getenv("PIPER_host"), "malware scanning host.") cmd.Flags().StringVar(&stepConfig.Username, "username", os.Getenv("PIPER_username"), "User") cmd.Flags().StringVar(&stepConfig.Password, "password", os.Getenv("PIPER_password"), "Password") - cmd.Flags().StringVar(&stepConfig.File, "file", os.Getenv("PIPER_file"), "The file which is scanned for malware") + cmd.Flags().StringVar(&stepConfig.ScanImage, "scanImage", os.Getenv("PIPER_scanImage"), "For `buildTool: docker`: Defines the docker image which should be scanned.") + cmd.Flags().BoolVar(&stepConfig.ScanImageIncludeLayers, "scanImageIncludeLayers", true, "For `buildTool: docker`: Defines if layers should be included.") + cmd.Flags().StringVar(&stepConfig.ScanImageRegistryURL, "scanImageRegistryUrl", os.Getenv("PIPER_scanImageRegistryUrl"), "For `buildTool: docker`: Defines the registry where the scanImage is located.") + cmd.Flags().StringVar(&stepConfig.ScanFile, "scanFile", os.Getenv("PIPER_scanFile"), "The file which is scanned for malware") cmd.Flags().StringVar(&stepConfig.Timeout, "timeout", `600`, "timeout for http layer in seconds") + cmd.Flags().StringVar(&stepConfig.ReportFileName, "reportFileName", `malwarescan_report.json`, "The file name of the report to be created") + cmd.MarkFlagRequired("buildTool") cmd.MarkFlagRequired("host") cmd.MarkFlagRequired("username") cmd.MarkFlagRequired("password") - cmd.MarkFlagRequired("file") } // retrieve step metadata @@ -132,7 +193,7 @@ func malwareExecuteScanMetadata() config.StepData { Metadata: config.StepMetadata{ Name: "malwareExecuteScan", Aliases: []config.Alias{}, - Description: "Performs a malware scan", + Description: "Performs a malware scan using the [SAP Malware Scanning Service](https://help.sap.com/viewer/b416237f818c4e2e827f6118640079f8/LATEST/en-US/b7c9b86fe724458086a502df3160f380.html).", }, Spec: config.StepSpec{ Inputs: config.StepInputs{ @@ -140,6 +201,73 @@ func malwareExecuteScanMetadata() config.StepData { {Name: "malwareScanCredentialsId", Description: "Jenkins 'Username with password' credentials ID containing the technical user/password credential used to communicate with the malwarescanning service.", Type: "jenkins"}, }, Parameters: []config.StepParameters{ + { + Name: "buildTool", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "buildTool", + }, + }, + Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: true, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_buildTool"), + }, + { + Name: "dockerConfigJSON", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "custom/dockerConfigJSON", + }, + + { + Name: "dockerConfigJsonCredentialsId", + Type: "secret", + }, + + { + Name: "dockerConfigFileVaultSecretName", + Type: "vaultSecretFile", + Default: "docker-config", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_dockerConfigJSON"), + }, + { + Name: "containerRegistryPassword", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "custom/repositoryPassword", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_containerRegistryPassword"), + }, + { + Name: "containerRegistryUser", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "custom/repositoryUsername", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_containerRegistryUser"), + }, { Name: "host", ResourceRef: []config.ResourceReference{}, @@ -192,13 +320,50 @@ func malwareExecuteScanMetadata() config.StepData { Default: os.Getenv("PIPER_password"), }, { - Name: "file", + Name: "scanImage", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/imageNameTag", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_scanImage"), + }, + { + Name: "scanImageIncludeLayers", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{}, + Default: true, + }, + { + Name: "scanImageRegistryUrl", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/registryUrl", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_scanImageRegistryUrl"), + }, + { + Name: "scanFile", ResourceRef: []config.ResourceReference{}, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", - Mandatory: true, - Aliases: []config.Alias{}, - Default: os.Getenv("PIPER_file"), + Mandatory: false, + Aliases: []config.Alias{{Name: "file", Deprecated: true}}, + Default: os.Getenv("PIPER_scanFile"), }, { Name: "timeout", @@ -209,6 +374,27 @@ func malwareExecuteScanMetadata() config.StepData { Aliases: []config.Alias{}, Default: `600`, }, + { + Name: "reportFileName", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: `malwarescan_report.json`, + }, + }, + }, + Outputs: config.StepOutputs{ + Resources: []config.StepResources{ + { + Name: "reports", + Type: "reports", + Parameters: []map[string]interface{}{ + {"filePattern": "**/toolrun_malwarescan_*.json", "type": "malwarescan"}, + {"type": "malwarescan"}, + }, + }, }, }, }, diff --git a/cmd/malwareExecuteScan_test.go b/cmd/malwareExecuteScan_test.go index 924fd8258..e02154f1b 100644 --- a/cmd/malwareExecuteScan_test.go +++ b/cmd/malwareExecuteScan_test.go @@ -2,11 +2,16 @@ package cmd import ( "fmt" + pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util" + piperDocker "github.com/SAP/jenkins-library/pkg/docker" piperhttp "github.com/SAP/jenkins-library/pkg/http" + "github.com/SAP/jenkins-library/pkg/malwarescan" + "github.com/SAP/jenkins-library/pkg/mock" "github.com/stretchr/testify/assert" "io" "io/ioutil" "net/http" + "os" "strings" "testing" ) @@ -15,186 +20,178 @@ var malwareScanConfig = malwareExecuteScanOptions{ Host: "https://example.org/malwarescanner", Username: "me", Password: "********", - File: "target/myFile", + ScanFile: "target/myFile", Timeout: "60", } -func TestMalwareScanTests(t *testing.T) { +type malwareScanUtilsMockBundle struct { + *mock.FilesMock - open = openFileMock() + returnScanResult *malwarescan.ScanResult + returnSHA256 string +} - getSHA256 = func(path string) (string, error) { - return "96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85", nil +func (utils *malwareScanUtilsMockBundle) SHA256(filePath string) (string, error) { + if utils.returnSHA256 == "" { + return utils.returnScanResult.SHA256, nil } - defer func() { - open = _open - getSHA256 = _getSHA256 - }() - t.Run("No malware, no encrypted content", func(t *testing.T) { - httpClient := httpMock{StatusCode: 200, ResponseBody: "{\"malwareDetected\":false,\"encryptedContentDetected\":false,\"scanSize\":298782,\"mimeType\":\"application/octet-stream\",\"SHA256\":\"96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85\"}"} + return utils.returnSHA256, nil +} - error := runMalwareScan(&malwareScanConfig, nil, nil, &httpClient) +func (utils *malwareScanUtilsMockBundle) OpenFile(path string, flag int, perm os.FileMode) (io.ReadCloser, error) { + return utils.FilesMock.Open(path, flag, perm) +} - if assert.NoError(t, error) { +func (utils *malwareScanUtilsMockBundle) FileWrite(path string, content []byte, perm os.FileMode) error { + return utils.FilesMock.FileWrite(path, content, perm) +} - t.Run("check url", func(t *testing.T) { - assert.Equal(t, "https://example.org/malwarescanner/scan", httpClient.URL) - }) +func (utils *malwareScanUtilsMockBundle) Info() (*malwarescan.Info, error) { + return &malwarescan.Info{EngineVersion: "Mock Malware Scanner", SignatureTimestamp: "n/a"}, nil +} - t.Run("check method", func(t *testing.T) { - assert.Equal(t, "POST", httpClient.Method) - }) +func (utils *malwareScanUtilsMockBundle) Scan(candidate io.Reader) (*malwarescan.ScanResult, error) { + return utils.returnScanResult, nil +} - t.Run("check user", func(t *testing.T) { - assert.Equal(t, "me", httpClient.Options.Username) - }) +func (utils *malwareScanUtilsMockBundle) newDockerClient(options piperDocker.ClientOptions) piperDocker.Download { + return &dockerClientMock{imageName: options.ImageName, registryURL: options.RegistryURL, localPath: options.LocalPath, includeLayers: options.IncludeLayers} +} - t.Run("check password", func(t *testing.T) { - assert.Equal(t, "********", httpClient.Options.Password) - }) +func TestMalwareScanWithNoBuildtool(t *testing.T) { + files := &mock.FilesMock{} + files.AddFile("target/myFile", []byte(`HELLO`)) + + utils := malwareScanUtilsMockBundle{ + FilesMock: files, + } + + t.Run("No malware, no encrypted content in file", func(t *testing.T) { + utils.returnScanResult = &malwarescan.ScanResult{ + MalwareDetected: false, + EncryptedContentDetected: false, + ScanSize: 298782, + MimeType: "application/octet-stream", + SHA256: "96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85", } + + error := runMalwareScan(&malwareScanConfig, nil, &utils) + + assert.NoError(t, error) }) - t.Run("Malware detected", func(t *testing.T) { - - httpClient := httpMock{StatusCode: 200, ResponseBody: "{\"malwareDetected\":true,\"encryptedContentDetected\":false,\"scanSize\":298782,\"mimeType\":\"application/octet-stream\",\"SHA256\":\"96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85\"}"} - error := runMalwareScan(&malwareScanConfig, nil, nil, &httpClient) - assert.EqualError(t, error, "Malware scan failed for file 'target/myFile'. Malware detected: true, encrypted content detected: false") - }) - - t.Run("Encrypted content detected", func(t *testing.T) { - - httpClient := httpMock{StatusCode: 200, ResponseBody: "{\"malwareDetected\": false, \"encryptedContentDetected\": true, \"scanSize\":298782,\"mimeType\":\"application/octet-stream\",\"SHA256\":\"96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85\"}"} - error := runMalwareScan(&malwareScanConfig, nil, nil, &httpClient) - assert.EqualError(t, error, "Malware scan failed for file 'target/myFile'. Malware detected: false, encrypted content detected: true") - }) - - t.Run("Malware and encrypted content detected", func(t *testing.T) { - - httpClient := httpMock{StatusCode: 200, ResponseBody: "{\"malwareDetected\": true, \"encryptedContentDetected\": true, \"scanSize\":298782,\"mimeType\":\"application/octet-stream\",\"SHA256\":\"96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85\"}"} - error := runMalwareScan(&malwareScanConfig, nil, nil, &httpClient) - assert.EqualError(t, error, "Malware scan failed for file 'target/myFile'. Malware detected: true, encrypted content detected: true") - }) -} - -func TestMalwareScanRequest(t *testing.T) { - - httpClient := httpMock{StatusCode: 200, ResponseBody: "{\"malwareDetected\":false,\"encryptedContentDetected\":false,\"scanSize\":298782,\"mimeType\":\"application/octet-stream\",\"SHA256\":\"96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85\"}"} - candidate := readCloserMock{Content: "HELLO"} - - malWareScanResponse, err := sendMalwareScanRequest(&httpClient, "POST", "https://example.org/malwarescanner", candidate) - - t.Run("Malware - scanRequest - check response body closed", func(t *testing.T) { - if assert.NoError(t, err) { - assert.True(t, httpClient.Body.Closed) + t.Run("Malware detected in file", func(t *testing.T) { + utils.returnScanResult = &malwarescan.ScanResult{ + MalwareDetected: true, + EncryptedContentDetected: false, + ScanSize: 298782, + MimeType: "application/octet-stream", + SHA256: "96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85", + Finding: "Win.Test.EICAR_HDB-1", } + + error := runMalwareScan(&malwareScanConfig, nil, &utils) + assert.EqualError(t, error, "Malware scan failed for file 'target/myFile'. Malware detected: true, encrypted content detected: false, finding: Win.Test.EICAR_HDB-1") }) - t.Run("Malware - scanRequest - check scan response", func(t *testing.T) { - - if assert.NoError(t, err) { - assert.False(t, malWareScanResponse.MalwareDetected) - assert.False(t, malWareScanResponse.EncryptedContentDetected) - assert.Equal(t, 298782, malWareScanResponse.ScanSize) - assert.Equal(t, "96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85", malWareScanResponse.SHA256) + t.Run("Encrypted content detected in file", func(t *testing.T) { + utils.returnScanResult = &malwarescan.ScanResult{ + MalwareDetected: false, + EncryptedContentDetected: true, + ScanSize: 298782, + MimeType: "application/octet-stream", + SHA256: "96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85", } - }) -} -func TestMalwareMarshalBody(t *testing.T) { - - t.Run("Malware - marshalResponse - body uninitialized", func(t *testing.T) { - var body []byte - r, error := marshalResponse(body) - assert.Nil(t, r) - assert.EqualError(t, error, "Unmarshalling of response body failed. Body: '': unexpected end of JSON input") - }) -} - -func TestMalwareValidateResponse(t *testing.T) { - - t.Run("Malware - validateResponse - HTTP error", func(t *testing.T) { - r, e := validateResponse(nil, fmt.Errorf("request error")) - assert.Nil(t, r) - assert.EqualError(t, e, "HTTP request failed with error: request error. Details: \"\"") - }) - t.Run("Malware - validateResponse - empty response", func(t *testing.T) { - r, e := validateResponse(&http.Response{}, fmt.Errorf("request error")) - assert.Nil(t, r) - assert.EqualError(t, e, "HTTP request failed with error: request error. Details: \"\"") + error := runMalwareScan(&malwareScanConfig, nil, &utils) + assert.EqualError(t, error, "Malware scan failed for file 'target/myFile'. Malware detected: false, encrypted content detected: true, finding: ") }) - t.Run("Malware - validateResponse - status code 500", func(t *testing.T) { - mock := responseMock(500, true) - r, e := validateResponse(mock, nil) - assert.Nil(t, r) - assert.True(t, strings.HasPrefix(e.Error(), "Unexpected response code (500)")) - }) - - t.Run("Malware - validateResponse - malwareDetected", func(t *testing.T) { - mock := responseMock(200, true) - r, e := validateResponse(mock, nil) - assert.NotNil(t, r) - assert.Nil(t, e) - }) - -} - -func responseMock(code int, malwareDetected bool) *http.Response { - if malwareDetected { - return &http.Response{StatusCode: code, Body: &readCloserMock{Content: "{\"malwareDetected\":true,\"encryptedContentDetected\":false,\"scanSize\":298782,\"mimeType\":\"application/octet-stream\",\"SHA256\":\"96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85\"}"}} - } - return &http.Response{StatusCode: code, Body: &readCloserMock{Content: "{\"malwareDetected\":false,\"encryptedContentDetected\":false,\"scanSize\":298782,\"mimeType\":\"application/octet-stream\",\"SHA256\":\"96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85\"}"}} -} - -func TestMalwareFileNotFound(t *testing.T) { - - httpClient := httpMock{StatusCode: 200} - - open = func(path string) (io.ReadCloser, error) { - return nil, fmt.Errorf("open %s: no such file or directory", path) - } - defer func() { open = _open }() - error := runMalwareScan(&malwareScanConfig, nil, nil, &httpClient) - - assert.Error(t, error, "open target/myFile: no such file or directory") -} - -func TestMalwareInternalServerError(t *testing.T) { - - httpClient := httpMock{StatusCode: 500, ResponseBody: "internal server error"} - - open = openFileMock() - - defer func() { open = _open }() - - err := runMalwareScan(&malwareScanConfig, nil, nil, &httpClient) - - assert.Error(t, err, "Unexpected response code (500). 200 expected. Details: \"internal server error\"") -} - -func TestMalwareRetrieveHash(t *testing.T) { - - open = openFileMock() - - defer func() { - open = _open - }() - hash, err := _getSHA256("target/myFile") - - if assert.NoError(t, err) { - assert.Equal(t, "3733cd977ff8eb18b987357e22ced99f46097f31ecb239e878ae63760e83e4d5", hash) - } -} - -func openFileMock() func(path string) (io.ReadCloser, error) { - return func(path string) (io.ReadCloser, error) { - if path == "target/myFile" { - return &readCloserMock{Content: "HELLO"}, nil // the content does not matter + t.Run("Malware and encrypted content detected in file", func(t *testing.T) { + utils.returnScanResult = &malwarescan.ScanResult{ + MalwareDetected: true, + EncryptedContentDetected: true, + ScanSize: 298782, + MimeType: "application/octet-stream", + SHA256: "96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85", + Finding: "Win.Test.EICAR_HDB-1", } - return nil, fmt.Errorf("open %s: no such file or directory", path) + + error := runMalwareScan(&malwareScanConfig, nil, &utils) + assert.EqualError(t, error, "Malware scan failed for file 'target/myFile'. Malware detected: true, encrypted content detected: true, finding: Win.Test.EICAR_HDB-1") + }) + + t.Run("No file and no buildtool specified", func(t *testing.T) { + malwareScanConfig := malwareExecuteScanOptions{ + Host: "https://example.org/malwarescanner", + Username: "me", + Password: "********", + Timeout: "60", + } + + error := runMalwareScan(&malwareScanConfig, nil, nil) + assert.EqualError(t, error, "Please specify a file to be scanned") + }) + + t.Run("File to be scanned, can't be found", func(t *testing.T) { + utils.returnScanResult = nil + + malwareScanConfig := malwareExecuteScanOptions{ + Host: "https://example.org/malwarescanner", + Username: "me", + Password: "********", + Timeout: "60", + ScanFile: "target/fileWhichDoesntExist", + } + + error := runMalwareScan(&malwareScanConfig, nil, &utils) + assert.EqualError(t, error, "the file 'target/fileWhichDoesntExist' does not exist: file does not exist") + }) +} + +func TestMalwareScanWithDockerAsBuildtoolTests(t *testing.T) { + files := &mock.FilesMock{} + files.AddFile("dockerimagename_latest.tar", []byte(``)) + + utils := malwareScanUtilsMockBundle{ + FilesMock: files, } + + t.Run("No malware detected in docker image", func(t *testing.T) { + utils.returnScanResult = &malwarescan.ScanResult{ + MalwareDetected: false, + EncryptedContentDetected: false, + ScanSize: 298782, + MimeType: "application/octet-stream", + SHA256: "96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85", + } + + malwareScanConfig := malwareExecuteScanOptions{ + Host: "https://example.org/malwarescanner", + Username: "me", + Password: "********", + Timeout: "60", + BuildTool: "docker", + ScanImage: "dockerimagename:latest", + } + + error := runMalwareScan(&malwareScanConfig, nil, &utils) + + assert.NoError(t, error) + }) + + t.Run("No file and no buildtool specified", func(t *testing.T) { + malwareScanConfig := malwareExecuteScanOptions{ + Host: "https://example.org/malwarescanner", + Username: "me", + Password: "********", + Timeout: "60", + } + + error := runMalwareScan(&malwareScanConfig, nil, &utils) + assert.EqualError(t, error, "Please specify a file to be scanned") + }) } type httpMock struct { @@ -211,6 +208,9 @@ func (c *httpMock) SetOptions(options piperhttp.ClientOptions) { } func (c *httpMock) SendRequest(method string, url string, r io.Reader, header http.Header, cookies []*http.Cookie) (*http.Response, error) { + if strings.HasSuffix(url, "/info") { + return &http.Response{StatusCode: 200, Body: &readCloserMock{Content: "{\"engineVersion\": \"Malware Service Mock\", \"signatureTimestamp\": \"2022-01-12T09:26:28.000Z\"}"}}, nil + } c.Method = method c.URL = url @@ -246,3 +246,24 @@ func (rc *readCloserMock) Close() error { rc.Closed = true return nil } + +type dockerClientMock struct { + imageName string + registryURL string + localPath string + includeLayers bool +} + +func (c *dockerClientMock) GetImageSource() (string, error) { + return c.imageName, nil +} + +//DownloadImageToPath download the image to the specified path +func (c *dockerClientMock) DownloadImageToPath(imageSource, filePath string) (pkgutil.Image, error) { + return pkgutil.Image{}, nil // fmt.Errorf("%s", filePath) +} + +//TarImage write a tar from the given image +func (c *dockerClientMock) TarImage(writer io.Writer, image pkgutil.Image) error { + return nil +} diff --git a/cmd/whitesourceExecuteScan.go b/cmd/whitesourceExecuteScan.go index 4a1ed7801..2a516b58b 100644 --- a/cmd/whitesourceExecuteScan.go +++ b/cmd/whitesourceExecuteScan.go @@ -176,7 +176,7 @@ func runWhitesourceScan(config *ScanOptions, scan *ws.Scan, utils whitesourceUti dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", IncludeLayers: saveImageOptions.IncludeLayers} dClient := &piperDocker.Client{} dClient.SetOptions(dClientOptions) - if err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient); err != nil { + if _, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient); err != nil { if strings.Contains(fmt.Sprint(err), "no image found") { log.SetErrorCategory(log.ErrorConfiguration) } diff --git a/pkg/malwarescan/malwarescan.go b/pkg/malwarescan/malwarescan.go new file mode 100644 index 000000000..5bc1fc139 --- /dev/null +++ b/pkg/malwarescan/malwarescan.go @@ -0,0 +1,123 @@ +package malwarescan + +import ( + "encoding/json" + "fmt" + piperhttp "github.com/SAP/jenkins-library/pkg/http" + "github.com/pkg/errors" + "io" + "io/ioutil" + "net/http" +) + +// ScanResult : Returned by the scan endpoint of the malwarescan api of SAP CP +type ScanResult struct { + MalwareDetected bool `json:"malwareDetected"` + EncryptedContentDetected bool `json:"encryptedContentDetected"` + ScanSize int `json:"scanSize"` + Finding string `json:"finding,omitempty"` + MimeType string `json:"mimeType"` + SHA256 string `json:"SHA256"` +} + +// Info : Returned by the info endpoint of the malwarescan api of SAP CP +type Info struct { + MaxScanSize int + SignatureTimestamp string + EngineVersion string +} + +// ScanError : Returned by the malwarescan api of SAP CP in case of an error +type ScanError struct { + Message string +} + +// Client : Interface for the malwarescan api provided by SAP CP (see https://api.sap.com/api/MalwareScanAPI/overview) +type Client interface { + Scan(candidate io.Reader) (*ScanResult, error) + Info() (*Info, error) +} + +// ClientImpl : Client implementation of the malwarescan api provided by SAP CP (see https://api.sap.com/api/MalwareScanAPI/overview) +type ClientImpl struct { + HTTPClient piperhttp.Sender + Host string +} + +// Scan : Triggers a malwarescan in SAP CP for the given content. +func (c *ClientImpl) Scan(candidate io.Reader) (*ScanResult, error) { + var scanResult ScanResult + + headers := http.Header{} + headers.Add("Content-Type", "application/octet-stream") + + err := c.sendAPIRequest("POST", "/scan", candidate, headers, &scanResult) + + if err != nil { + return nil, err + } + + return &scanResult, nil + +} + +// Info : Returns some information about the scanengine used by the malwarescan service. +func (c *ClientImpl) Info() (*Info, error) { + var info Info + + err := c.sendAPIRequest("GET", "/info", nil, nil, &info) + + if err != nil { + return nil, err + } + + return &info, nil +} + +func (c *ClientImpl) sendAPIRequest(method, endpoint string, body io.Reader, header http.Header, obj interface{}) error { + // piper http utils mashall some http response codes into errors. We wan't to check the status code + // ourselves hence we wait with returning that error (maybe also related to errors others than http status codes) + + // sendRequest results in any combination of nil and non-nil response and error. + // a response body could even be already closed. + response, err := c.HTTPClient.SendRequest(method, c.Host+endpoint, body, header, nil) + + if response.StatusCode != 200 { + var scanError ScanError + + err = c.unmarshalResponse(response, &scanError) + + if err != nil { + return fmt.Errorf("MalwareService returned with status code %d, no further information available", response.StatusCode) + } + + return fmt.Errorf("MalwareService returned with status code %d: %s", response.StatusCode, scanError.Message) + } + + return c.unmarshalResponse(response, obj) +} + +func (c *ClientImpl) readBody(response *http.Response) ([]byte, error) { + if response != nil && response.Body != nil { + defer response.Body.Close() + return ioutil.ReadAll(response.Body) + } + + return nil, fmt.Errorf("No response body available") +} + +func (c *ClientImpl) unmarshalResponse(response *http.Response, obj interface{}) error { + body, err := c.readBody(response) + + if err != nil { + return err + } + + err = json.Unmarshal(body, obj) + + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Unmarshalling of response body failed. Body: '%s'", body)) + } + + return err +} diff --git a/pkg/malwarescan/malwarescan_test.go b/pkg/malwarescan/malwarescan_test.go new file mode 100644 index 000000000..733ba0503 --- /dev/null +++ b/pkg/malwarescan/malwarescan_test.go @@ -0,0 +1,179 @@ +package malwarescan + +import ( + "fmt" + piperhttp "github.com/SAP/jenkins-library/pkg/http" + "github.com/stretchr/testify/assert" + "io" + "io/ioutil" + "net/http" + "testing" +) + +func TestMalwareServiceScan(t *testing.T) { + t.Run("Scan without finding", func(t *testing.T) { + httpClient := &httpMock{StatusCode: 200, ResponseBody: "{\"malwareDetected\":false,\"encryptedContentDetected\":false,\"scanSize\":298782,\"mimeType\":\"application/octet-stream\",\"SHA256\":\"96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85\"}"} + + malwareService := ClientImpl{ + HTTPClient: httpClient, + Host: "https://example.org/malwarescanner", + } + + candidate := readCloserMock{Content: "HELLO"} + scanResult, err := malwareService.Scan(candidate) + + if assert.NoError(t, err) { + assert.True(t, httpClient.Body.Closed) + + assert.Equal(t, "https://example.org/malwarescanner/scan", httpClient.URL) + assert.Equal(t, "POST", httpClient.Method) + + if assert.NotNil(t, httpClient.Header) { + assert.Equal(t, "application/octet-stream", httpClient.Header.Get("Content-Type")) + } + + assert.Equal(t, "application/octet-stream", scanResult.MimeType) + assert.Equal(t, 298782, scanResult.ScanSize) + assert.Equal(t, "96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85", scanResult.SHA256) + assert.Equal(t, "", scanResult.Finding) + assert.False(t, scanResult.MalwareDetected) + assert.False(t, scanResult.EncryptedContentDetected) + } + }) + + t.Run("Scan without finding", func(t *testing.T) { + httpClient := &httpMock{StatusCode: 200, ResponseBody: "{\"malwareDetected\":true,\"encryptedContentDetected\":true,\"scanSize\":298782,\"mimeType\":\"application/octet-stream\",\"SHA256\":\"96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85\", \"finding\": \"Description of the finding\"}"} + + malwareService := ClientImpl{ + HTTPClient: httpClient, + Host: "https://example.org/malwarescanner", + } + + candidate := readCloserMock{Content: "HELLO"} + scanResult, err := malwareService.Scan(candidate) + + if assert.NoError(t, err) { + assert.True(t, httpClient.Body.Closed) + + assert.Equal(t, "https://example.org/malwarescanner/scan", httpClient.URL) + assert.Equal(t, "POST", httpClient.Method) + + if assert.NotNil(t, httpClient.Header) { + assert.Equal(t, "application/octet-stream", httpClient.Header.Get("Content-Type")) + } + + assert.Equal(t, "application/octet-stream", scanResult.MimeType) + assert.Equal(t, 298782, scanResult.ScanSize) + assert.Equal(t, "96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85", scanResult.SHA256) + assert.Equal(t, "Description of the finding", scanResult.Finding) + assert.True(t, scanResult.MalwareDetected) + assert.True(t, scanResult.EncryptedContentDetected) + } + }) + + t.Run("Scan results in error - file to large", func(t *testing.T) { + httpClient := &httpMock{StatusCode: 413, ResponseBody: "{\"message\":\"Payload too large - The file is too large and cannot be scanned or the archive structure is too complex.\"}"} + + malwareService := ClientImpl{ + HTTPClient: httpClient, + Host: "https://example.org/malwarescanner", + } + + candidate := readCloserMock{Content: "HELLO"} + scanResult, err := malwareService.Scan(candidate) + + assert.Nil(t, scanResult) + assert.EqualError(t, err, "MalwareService returned with status code 413: Payload too large - The file is too large and cannot be scanned or the archive structure is too complex.") + }) + + t.Run("Scan results in error - unexpected error", func(t *testing.T) { + httpClient := &httpMock{StatusCode: 500, ResponseBody: ""} + + malwareService := ClientImpl{ + HTTPClient: httpClient, + Host: "https://example.org/malwarescanner", + } + + candidate := readCloserMock{Content: "HELLO"} + scanResult, err := malwareService.Scan(candidate) + + assert.Nil(t, scanResult) + assert.EqualError(t, err, "MalwareService returned with status code 500, no further information available") + }) +} + +func TestMalwareServiceInfo(t *testing.T) { + t.Run("Receives engine info", func(t *testing.T) { + httpClient := &httpMock{StatusCode: 200, ResponseBody: "{\"engineVersion\": \"Malware Service Mock\", \"signatureTimestamp\": \"2022-01-12T09:26:28.000Z\", \"maxScanSize\": 666}"} + + malwareService := ClientImpl{ + HTTPClient: httpClient, + Host: "https://example.org/malwarescanner", + } + + info, err := malwareService.Info() + + if assert.NoError(t, err) { + assert.True(t, httpClient.Body.Closed) + + assert.Equal(t, "https://example.org/malwarescanner/info", httpClient.URL) + assert.Equal(t, "GET", httpClient.Method) + assert.Equal(t, "Malware Service Mock", info.EngineVersion) + assert.Equal(t, "2022-01-12T09:26:28.000Z", info.SignatureTimestamp) + assert.Equal(t, 666, info.MaxScanSize) + } + }) +} + +type httpMock struct { + Method string // is set during test execution + URL string // is set before test execution + ResponseBody string // is set before test execution + Options piperhttp.ClientOptions // is set during test + StatusCode int // is set during test + Body readCloserMock // is set during test + Header http.Header // is set during test +} + +func (c *httpMock) SetOptions(options piperhttp.ClientOptions) { + c.Options = options +} + +func (c *httpMock) SendRequest(method string, url string, r io.Reader, header http.Header, cookies []*http.Cookie) (*http.Response, error) { + c.Method = method + c.URL = url + c.Header = header + + if r != nil { + _, err := ioutil.ReadAll(r) + + if err != nil { + return nil, err + } + } + + c.Body = readCloserMock{Content: c.ResponseBody} + res := http.Response{StatusCode: c.StatusCode, Body: &c.Body} + + return &res, nil +} + +type readCloserMock struct { + Content string + Closed bool +} + +func (rc readCloserMock) Read(b []byte) (n int, err error) { + + if len(b) < len(rc.Content) { + // in real life we would fill the buffer according to buffer size ... + return 0, fmt.Errorf("Buffer size (%d) not sufficient, need: %d", len(b), len(rc.Content)) + } + copy(b, rc.Content) + return len(rc.Content), io.EOF +} + +func (rc *readCloserMock) Close() error { + rc.Closed = true + return nil +} diff --git a/pkg/mock/fileUtils.go b/pkg/mock/fileUtils.go index f29bb8bea..0d8ac8690 100644 --- a/pkg/mock/fileUtils.go +++ b/pkg/mock/fileUtils.go @@ -505,6 +505,15 @@ type FileMock struct { content []byte } +// Reads the content of the mock +func (f *FileMock) Read(b []byte) (n int, err error) { + for i, p := range f.content { + b[i] = p + } + + return len(f.content), nil +} + // Close mocks freeing the associated OS resources. func (f *FileMock) Close() error { f.files = nil @@ -542,7 +551,7 @@ func (f *FileMock) Write(p []byte) (n int, err error) { // Instead, it returns a pointer to a FileMock instance, which implements a number of the same methods as os.File. // The flag parameter is checked for os.O_CREATE and os.O_APPEND and behaves accordingly. func (f *FilesMock) Open(path string, flag int, perm os.FileMode) (*FileMock, error) { - if f.files == nil && flag&os.O_CREATE == 0 { + if (f.files == nil || !f.HasFile(path)) && flag&os.O_CREATE == 0 { return nil, fmt.Errorf("the file '%s' does not exist: %w", path, os.ErrNotExist) } f.init() diff --git a/pkg/piperutils/FileUtils.go b/pkg/piperutils/FileUtils.go index 7bff3e59c..0310b0882 100644 --- a/pkg/piperutils/FileUtils.go +++ b/pkg/piperutils/FileUtils.go @@ -4,6 +4,7 @@ import ( "archive/tar" "archive/zip" "compress/gzip" + "crypto/sha256" "errors" "fmt" "io" @@ -390,3 +391,20 @@ func (f Files) Abs(path string) (string, error) { func (f Files) Symlink(oldname, newname string) error { return os.Symlink(oldname, newname) } + +// Computes a SHA256 for a given file +func (f Files) SHA256(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + + hash := sha256.New() + _, err = io.Copy(hash, file) + if err != nil { + return "", err + } + + return fmt.Sprintf("%x", string(hash.Sum(nil))), nil +} diff --git a/resources/metadata/malwareExecuteScan.yaml b/resources/metadata/malwareExecuteScan.yaml index 63b205b89..43f74c96e 100644 --- a/resources/metadata/malwareExecuteScan.yaml +++ b/resources/metadata/malwareExecuteScan.yaml @@ -1,8 +1,8 @@ metadata: name: malwareExecuteScan - description: Performs a malware scan + description: Performs a malware scan using the [SAP Malware Scanning Service](https://help.sap.com/viewer/b416237f818c4e2e827f6118640079f8/LATEST/en-US/b7c9b86fe724458086a502df3160f380.html). longDescription: | - Performs a malware scan + Performs a malware scan using the [SAP Malware Scanning Service](https://help.sap.com/viewer/b416237f818c4e2e827f6118640079f8/LATEST/en-US/b7c9b86fe724458086a502df3160f380.html). spec: inputs: secrets: @@ -10,6 +10,56 @@ spec: description: Jenkins 'Username with password' credentials ID containing the technical user/password credential used to communicate with the malwarescanning service. type: jenkins params: + - name: buildTool + type: string + description: "Defines the tool which is used for building the artifact." + mandatory: true + scope: + - GENERAL + - PARAMETERS + - STAGES + - STEPS + resourceRef: + - name: commonPipelineEnvironment + param: buildTool + - name: dockerConfigJSON + type: string + description: Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/). + scope: + - PARAMETERS + - STAGES + - STEPS + secret: true + resourceRef: + - name: commonPipelineEnvironment + param: custom/dockerConfigJSON + - name: dockerConfigJsonCredentialsId + type: secret + - type: vaultSecretFile + name: dockerConfigFileVaultSecretName + default: docker-config + - name: containerRegistryPassword + description: "For `buildTool: docker`: Password for container registry access - typically provided by the CI/CD environment." + type: string + scope: + - PARAMETERS + - STAGES + - STEPS + secret: true + resourceRef: + - name: commonPipelineEnvironment + param: custom/repositoryPassword + - name: containerRegistryUser + description: "For `buildTool: docker`: Username for container registry access - typically provided by the CI/CD environment." + type: string + scope: + - PARAMETERS + - STAGES + - STEPS + secret: true + resourceRef: + - name: commonPipelineEnvironment + param: custom/repositoryUsername - name: host type: string description: "malware scanning host." @@ -50,14 +100,44 @@ spec: - name: malwareScanPasswordVaultSecretName type: vaultSecret default: malware-scan - - name: file + - name: scanImage + type: string + description: "For `buildTool: docker`: Defines the docker image which should be scanned." + resourceRef: + - name: commonPipelineEnvironment + param: container/imageNameTag + scope: + - PARAMETERS + - STAGES + - STEPS + - name: scanImageIncludeLayers + type: bool + description: "For `buildTool: docker`: Defines if layers should be included." + scope: + - PARAMETERS + - STAGES + - STEPS + default: true + - name: scanImageRegistryUrl + type: string + description: "For `buildTool: docker`: Defines the registry where the scanImage is located." + resourceRef: + - name: commonPipelineEnvironment + param: container/registryUrl + scope: + - PARAMETERS + - STAGES + - STEPS + - name: scanFile + aliases: + - name: file + deprecated: true type: string description: "The file which is scanned for malware" scope: - PARAMETERS - STAGES - STEPS - mandatory: true - name: timeout type: string description: "timeout for http layer in seconds" @@ -67,3 +147,20 @@ spec: - STEPS mandatory: false default: 600 + - name: reportFileName + type: string + description: The file name of the report to be created + scope: + - PARAMETERS + - STAGES + - STEPS + default: malwarescan_report.json + outputs: + resources: + - name: reports + type: reports + params: + - filePattern: "**/toolrun_malwarescan_*.json" + type: malwarescan + - paramRef: reportFileName + type: malwarescan