You've already forked sap-jenkins-library
							
							
				mirror of
				https://github.com/SAP/jenkins-library.git
				synced 2025-10-30 23:57:50 +02:00 
			
		
		
		
	feat(docker): use crane for pulling docker images (#3652)
This commit is contained in:
		| @@ -4,7 +4,7 @@ import ( | ||||
| 	"fmt" | ||||
| 	"os" | ||||
| 	"path/filepath" | ||||
| 	"strings" | ||||
| 	"regexp" | ||||
|  | ||||
| 	piperDocker "github.com/SAP/jenkins-library/pkg/docker" | ||||
| 	"github.com/SAP/jenkins-library/pkg/log" | ||||
| @@ -17,11 +17,13 @@ import ( | ||||
| func containerSaveImage(config containerSaveImageOptions, telemetryData *telemetry.CustomData) { | ||||
| 	var cachePath = "./cache" | ||||
|  | ||||
| 	dClientOptions := piperDocker.ClientOptions{ImageName: config.ContainerImage, RegistryURL: config.ContainerRegistryURL, LocalPath: config.FilePath, IncludeLayers: config.IncludeLayers} | ||||
| 	fileUtils := piperutils.Files{} | ||||
|  | ||||
| 	dClientOptions := piperDocker.ClientOptions{ImageName: config.ContainerImage, RegistryURL: config.ContainerRegistryURL, LocalPath: config.FilePath} | ||||
| 	dClient := &piperDocker.Client{} | ||||
| 	dClient.SetOptions(dClientOptions) | ||||
|  | ||||
| 	_, err := runContainerSaveImage(&config, telemetryData, cachePath, "", dClient, piperutils.Files{}) | ||||
| 	_, err := runContainerSaveImage(&config, telemetryData, cachePath, "", dClient, fileUtils) | ||||
| 	if err != nil { | ||||
| 		log.Entry().WithError(err).Fatal("step execution failed") | ||||
| 	} | ||||
| @@ -32,55 +34,26 @@ func runContainerSaveImage(config *containerSaveImageOptions, telemetryData *tel | ||||
| 		return "", err | ||||
| 	} | ||||
|  | ||||
| 	err := os.RemoveAll(cachePath) | ||||
| 	if err != nil { | ||||
| 		return "", errors.Wrap(err, "failed to prepare cache") | ||||
| 	} | ||||
|  | ||||
| 	err = os.Mkdir(cachePath, 0755) | ||||
| 	if err != nil { | ||||
| 		return "", errors.Wrap(err, "failed to create cache") | ||||
| 	} | ||||
|  | ||||
| 	// ensure that download cache is cleaned up at the end | ||||
| 	defer os.RemoveAll(cachePath) | ||||
|  | ||||
| 	imageSource, err := dClient.GetImageSource() | ||||
| 	if err != nil { | ||||
| 		return "", errors.Wrap(err, "failed to get docker image source") | ||||
| 	} | ||||
| 	image, err := dClient.DownloadImageToPath(imageSource, cachePath) | ||||
| 	if err != nil { | ||||
| 		return "", errors.Wrap(err, "failed to download docker image") | ||||
| 	} | ||||
|  | ||||
| 	tarfilePath := config.FilePath | ||||
|  | ||||
| 	if len(tarfilePath) == 0 { | ||||
| 		tarfilePath = filenameFromContainer(rootPath, config.ContainerImage) | ||||
| 	} else { | ||||
| 		tarfilePath = filenameFromContainer(rootPath, tarfilePath) | ||||
| 		tarfilePath = filepath.Join(rootPath, tarfilePath) | ||||
| 	} | ||||
|  | ||||
| 	tarFile, err := os.Create(tarfilePath) | ||||
| 	if err != nil { | ||||
| 		return "", errors.Wrapf(err, "failed to create %v for docker image", tarfilePath) | ||||
| 	} | ||||
| 	defer tarFile.Close() | ||||
|  | ||||
| 	if err := os.Chmod(tarfilePath, 0644); err != nil { | ||||
| 		return "", errors.Wrapf(err, "failed to adapt permissions on %v", tarfilePath) | ||||
| 	} | ||||
|  | ||||
| 	err = dClient.TarImage(tarFile, image) | ||||
| 	if err != nil { | ||||
| 		return "", errors.Wrap(err, "failed to tar container image") | ||||
| 	log.Entry().Infof("Downloading '%s' to '%s'", config.ContainerImage, tarfilePath) | ||||
| 	if _, err := dClient.DownloadImage(config.ContainerImage, tarfilePath); err != nil { | ||||
| 		return "", errors.Wrap(err, "failed to download docker image") | ||||
| 	} | ||||
|  | ||||
| 	return tarfilePath, nil | ||||
| } | ||||
|  | ||||
| func filenameFromContainer(rootPath, containerImage string) string { | ||||
| 	return filepath.Join(rootPath, strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(containerImage, "/", "_"), ":", "_"), ".", "_")+".tar") | ||||
| 	re := regexp.MustCompile("[^a-zA-Z0-9-]") | ||||
|  | ||||
| 	return filepath.Join(rootPath, fmt.Sprintf("%s.tar", re.ReplaceAllString(containerImage, "_"))) | ||||
| } | ||||
|  | ||||
| func correctContainerDockerConfigEnvVar(config *containerSaveImageOptions, utils piperutils.FileUtils) error { | ||||
|   | ||||
| @@ -21,7 +21,6 @@ type containerSaveImageOptions struct { | ||||
| 	ContainerRegistryPassword string `json:"containerRegistryPassword,omitempty"` | ||||
| 	ContainerRegistryUser     string `json:"containerRegistryUser,omitempty"` | ||||
| 	FilePath                  string `json:"filePath,omitempty"` | ||||
| 	IncludeLayers             bool   `json:"includeLayers,omitempty"` | ||||
| 	DockerConfigJSON          string `json:"dockerConfigJSON,omitempty"` | ||||
| } | ||||
|  | ||||
| @@ -119,12 +118,11 @@ It can be used no matter if a Docker daemon is available or not. It will also wo | ||||
| } | ||||
|  | ||||
| func addContainerSaveImageFlags(cmd *cobra.Command, stepConfig *containerSaveImageOptions) { | ||||
| 	cmd.Flags().StringVar(&stepConfig.ContainerRegistryURL, "containerRegistryUrl", os.Getenv("PIPER_containerRegistryUrl"), "The reference to the container registry where the image is located.") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ContainerRegistryURL, "containerRegistryUrl", os.Getenv("PIPER_containerRegistryUrl"), "For `buildTool: docker`: Url of the container registry - typically provided by the CI/CD environment.") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ContainerImage, "containerImage", os.Getenv("PIPER_containerImage"), "Container image to be saved.") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ContainerRegistryPassword, "containerRegistryPassword", os.Getenv("PIPER_containerRegistryPassword"), "For `buildTool: docker`: Password for container registry access - typically provided by the CI/CD environment.") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ContainerRegistryUser, "containerRegistryUser", os.Getenv("PIPER_containerRegistryUser"), "For `buildTool: docker`: Username for container registry access - typically provided by the CI/CD environment.") | ||||
| 	cmd.Flags().StringVar(&stepConfig.FilePath, "filePath", os.Getenv("PIPER_filePath"), "The path to the file to which the image should be saved. Defaults to `containerImage.tar`") | ||||
| 	cmd.Flags().BoolVar(&stepConfig.IncludeLayers, "includeLayers", false, "Flag if the docker layers should be included") | ||||
| 	cmd.Flags().StringVar(&stepConfig.FilePath, "filePath", os.Getenv("PIPER_filePath"), "The path to the file to which the image should be saved.") | ||||
| 	cmd.Flags().StringVar(&stepConfig.DockerConfigJSON, "dockerConfigJSON", os.Getenv("PIPER_dockerConfigJSON"), "Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).") | ||||
|  | ||||
| 	cmd.MarkFlagRequired("containerRegistryUrl") | ||||
| @@ -220,15 +218,6 @@ func containerSaveImageMetadata() config.StepData { | ||||
| 						Aliases:     []config.Alias{}, | ||||
| 						Default:     os.Getenv("PIPER_filePath"), | ||||
| 					}, | ||||
| 					{ | ||||
| 						Name:        "includeLayers", | ||||
| 						ResourceRef: []config.ResourceReference{}, | ||||
| 						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"}, | ||||
| 						Type:        "bool", | ||||
| 						Mandatory:   false, | ||||
| 						Aliases:     []config.Alias{}, | ||||
| 						Default:     false, | ||||
| 					}, | ||||
| 					{ | ||||
| 						Name: "dockerConfigJSON", | ||||
| 						ResourceRef: []config.ResourceReference{ | ||||
|   | ||||
| @@ -2,106 +2,47 @@ package cmd | ||||
|  | ||||
| import ( | ||||
| 	"fmt" | ||||
| 	"io" | ||||
| 	"io/ioutil" | ||||
| 	"os" | ||||
| 	"path/filepath" | ||||
| 	"testing" | ||||
|  | ||||
| 	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util" | ||||
| 	"github.com/SAP/jenkins-library/pkg/mock" | ||||
| 	"github.com/SAP/jenkins-library/pkg/telemetry" | ||||
| 	"github.com/stretchr/testify/assert" | ||||
|  | ||||
| 	"github.com/google/go-containerregistry/pkg/v1" | ||||
| 	"github.com/google/go-containerregistry/pkg/v1/fake" | ||||
| ) | ||||
|  | ||||
| type containerMock struct { | ||||
| 	filePath         string | ||||
| 	imageSource      string | ||||
| 	registryURL      string | ||||
| 	localPath        string | ||||
| 	includeLayers    bool | ||||
| 	downloadImageErr string | ||||
| 	imageSourceErr   string | ||||
| 	tarImageErr      string | ||||
| } | ||||
|  | ||||
| func (c *containerMock) DownloadImageToPath(imageSource, filePath string) (pkgutil.Image, error) { | ||||
| 	c.imageSource = imageSource | ||||
| 	c.filePath = filePath | ||||
| 	if c.downloadImageErr != "" { | ||||
| 		return pkgutil.Image{}, fmt.Errorf(c.downloadImageErr) | ||||
| 	} | ||||
| 	return pkgutil.Image{}, nil | ||||
| } | ||||
|  | ||||
| func (c *containerMock) GetImageSource() (string, error) { | ||||
| 	if c.imageSourceErr != "" { | ||||
| 		return "", fmt.Errorf(c.imageSourceErr) | ||||
| 	} | ||||
| 	return "imageSource", nil | ||||
| } | ||||
|  | ||||
| func (c *containerMock) TarImage(writer io.Writer, image pkgutil.Image) error { | ||||
| 	if c.tarImageErr != "" { | ||||
| 		return fmt.Errorf(c.tarImageErr) | ||||
| 	} | ||||
| 	writer.Write([]byte("This is a test")) | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| func TestRunContainerSaveImage(t *testing.T) { | ||||
| 	telemetryData := telemetry.CustomData{} | ||||
|  | ||||
| 	t.Run("success case", func(t *testing.T) { | ||||
| 		config := containerSaveImageOptions{} | ||||
| 		tmpFolder, err := ioutil.TempDir("", "") | ||||
| 		if err != nil { | ||||
| 			t.Fatal("failed to create temp dir") | ||||
| 		} | ||||
| 		defer os.RemoveAll(tmpFolder) | ||||
| 		config.FilePath = "testfile.tar" | ||||
|  | ||||
| 		cacheFolder := filepath.Join(tmpFolder, "cache") | ||||
|  | ||||
| 		config.FilePath = "testfile" | ||||
|  | ||||
| 		dClient := containerMock{} | ||||
| 		dClient := mock.DownloadMock{} | ||||
| 		files := mock.FilesMock{} | ||||
|  | ||||
| 		filePath, err := runContainerSaveImage(&config, &telemetryData, cacheFolder, tmpFolder, &dClient, &files) | ||||
| 		cacheFolder, err := files.TempDir("", "containerSaveImage-") | ||||
| 		assert.NoError(t, err) | ||||
|  | ||||
| 		assert.Equal(t, cacheFolder, dClient.filePath) | ||||
| 		assert.Equal(t, "imageSource", dClient.imageSource) | ||||
| 		dClient.Stub = func(imgRef string, dest string) (v1.Image, error) { | ||||
| 			files.AddFile(dest, []byte("This is a test")) | ||||
| 			return &fake.FakeImage{}, nil | ||||
| 		} | ||||
|  | ||||
| 		content, err := ioutil.ReadFile(filepath.Join(tmpFolder, "testfile.tar")) | ||||
| 		filePath, err := runContainerSaveImage(&config, &telemetryData, cacheFolder, cacheFolder, &dClient, &files) | ||||
| 		assert.NoError(t, err) | ||||
|  | ||||
| 		content, err := files.FileRead(filepath.Join(cacheFolder, "testfile.tar")) | ||||
| 		assert.NoError(t, err) | ||||
| 		assert.Equal(t, "This is a test", string(content)) | ||||
|  | ||||
| 		assert.Contains(t, filePath, "testfile.tar") | ||||
| 	}) | ||||
|  | ||||
| 	t.Run("failure - cache creation", func(t *testing.T) { | ||||
| 		config := containerSaveImageOptions{} | ||||
| 		dClient := containerMock{} | ||||
| 		files := mock.FilesMock{} | ||||
| 		_, err := runContainerSaveImage(&config, &telemetryData, "", "", &dClient, &files) | ||||
| 		assert.Contains(t, fmt.Sprint(err), "failed to create cache: mkdir :") | ||||
| 	}) | ||||
|  | ||||
| 	t.Run("failure - get image source", func(t *testing.T) { | ||||
| 		config := containerSaveImageOptions{} | ||||
| 		tmpFolder, err := ioutil.TempDir("", "") | ||||
| 		if err != nil { | ||||
| 			t.Fatal("failed to create temp dir") | ||||
| 		} | ||||
| 		defer os.RemoveAll(tmpFolder) | ||||
|  | ||||
| 		dClient := containerMock{imageSourceErr: "image source error"} | ||||
| 		files := mock.FilesMock{} | ||||
| 		_, err = runContainerSaveImage(&config, &telemetryData, filepath.Join(tmpFolder, "cache"), tmpFolder, &dClient, &files) | ||||
| 		assert.EqualError(t, err, "failed to get docker image source: image source error") | ||||
| 	}) | ||||
|  | ||||
| 	t.Run("failure - download image", func(t *testing.T) { | ||||
| 		config := containerSaveImageOptions{} | ||||
| 		tmpFolder, err := ioutil.TempDir("", "") | ||||
| @@ -110,25 +51,11 @@ func TestRunContainerSaveImage(t *testing.T) { | ||||
| 		} | ||||
| 		defer os.RemoveAll(tmpFolder) | ||||
|  | ||||
| 		dClient := containerMock{downloadImageErr: "download error"} | ||||
| 		dClient := mock.DownloadMock{ReturnError: "download error"} | ||||
| 		files := mock.FilesMock{} | ||||
| 		_, err = runContainerSaveImage(&config, &telemetryData, filepath.Join(tmpFolder, "cache"), tmpFolder, &dClient, &files) | ||||
| 		assert.EqualError(t, err, "failed to download docker image: download error") | ||||
| 	}) | ||||
|  | ||||
| 	t.Run("failure - tar image", func(t *testing.T) { | ||||
| 		config := containerSaveImageOptions{} | ||||
| 		tmpFolder, err := ioutil.TempDir("", "") | ||||
| 		if err != nil { | ||||
| 			t.Fatal("failed to create temp dir") | ||||
| 		} | ||||
| 		defer os.RemoveAll(tmpFolder) | ||||
|  | ||||
| 		dClient := containerMock{tarImageErr: "tar error"} | ||||
| 		files := mock.FilesMock{} | ||||
| 		_, err = runContainerSaveImage(&config, &telemetryData, filepath.Join(tmpFolder, "cache"), tmpFolder, &dClient, &files) | ||||
| 		assert.EqualError(t, err, "failed to tar container image: tar error") | ||||
| 	}) | ||||
| } | ||||
|  | ||||
| func TestFilenameFromContainer(t *testing.T) { | ||||
|   | ||||
| @@ -5,7 +5,6 @@ import ( | ||||
| 	"fmt" | ||||
| 	"io" | ||||
| 	"net/http" | ||||
| 	"os" | ||||
| 	"path/filepath" | ||||
| 	"regexp" | ||||
| 	"sort" | ||||
| @@ -29,25 +28,7 @@ import ( | ||||
| ) | ||||
|  | ||||
| type detectUtils interface { | ||||
| 	Abs(path string) (string, error) | ||||
| 	FileExists(filename string) (bool, error) | ||||
| 	FileRemove(filename string) error | ||||
| 	Copy(src, dest string) (int64, error) | ||||
| 	Move(src, dest string) error | ||||
| 	DirExists(dest string) (bool, error) | ||||
| 	FileRead(path string) ([]byte, error) | ||||
| 	FileWrite(path string, content []byte, perm os.FileMode) error | ||||
| 	MkdirAll(path string, perm os.FileMode) error | ||||
| 	Chmod(path string, mode os.FileMode) error | ||||
| 	Glob(pattern string) (matches []string, err error) | ||||
| 	Chdir(path string) error | ||||
| 	TempDir(string, string) (string, error) | ||||
| 	RemoveAll(string) error | ||||
| 	FileRename(string, string) error | ||||
| 	Getwd() (string, error) | ||||
| 	Symlink(oldname string, newname string) error | ||||
| 	SHA256(path string) (string, error) | ||||
| 	CurrentTime(format string) string | ||||
| 	piperutils.FileUtils | ||||
|  | ||||
| 	GetExitCode() int | ||||
| 	GetOsEnv() []string | ||||
|   | ||||
| @@ -150,10 +150,9 @@ func selectAndPrepareFileForMalwareScan(config *malwareExecuteScanOptions, utils | ||||
| 			ContainerRegistryUser:     config.ContainerRegistryUser, | ||||
| 			ContainerRegistryPassword: config.ContainerRegistryPassword, | ||||
| 			DockerConfigJSON:          config.DockerConfigJSON, | ||||
| 			IncludeLayers:             config.ScanImageIncludeLayers, | ||||
| 			FilePath:                  config.ScanImage, | ||||
| 		} | ||||
| 		dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", IncludeLayers: saveImageOptions.IncludeLayers} | ||||
|  | ||||
| 		dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: ""} | ||||
| 		dClient := utils.newDockerClient(dClientOptions) | ||||
|  | ||||
| 		tarFile, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient, utils) | ||||
|   | ||||
| @@ -28,7 +28,6 @@ type malwareExecuteScanOptions struct { | ||||
| 	Username                  string `json:"username,omitempty"` | ||||
| 	Password                  string `json:"password,omitempty"` | ||||
| 	ScanImage                 string `json:"scanImage,omitempty"` | ||||
| 	ScanImageIncludeLayers    bool   `json:"scanImageIncludeLayers,omitempty"` | ||||
| 	ScanImageRegistryURL      string `json:"scanImageRegistryUrl,omitempty"` | ||||
| 	ScanFile                  string `json:"scanFile,omitempty"` | ||||
| 	Timeout                   string `json:"timeout,omitempty"` | ||||
| @@ -176,7 +175,6 @@ func addMalwareExecuteScanFlags(cmd *cobra.Command, stepConfig *malwareExecuteSc | ||||
| 	cmd.Flags().StringVar(&stepConfig.Username, "username", os.Getenv("PIPER_username"), "User") | ||||
| 	cmd.Flags().StringVar(&stepConfig.Password, "password", os.Getenv("PIPER_password"), "Password") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ScanImage, "scanImage", os.Getenv("PIPER_scanImage"), "For `buildTool: docker`: Defines the docker image which should be scanned.") | ||||
| 	cmd.Flags().BoolVar(&stepConfig.ScanImageIncludeLayers, "scanImageIncludeLayers", true, "For `buildTool: docker`: Defines if layers should be included.") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ScanImageRegistryURL, "scanImageRegistryUrl", os.Getenv("PIPER_scanImageRegistryUrl"), "For `buildTool: docker`: Defines the registry where the scanImage is located.") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ScanFile, "scanFile", os.Getenv("PIPER_scanFile"), "The file which is scanned for malware") | ||||
| 	cmd.Flags().StringVar(&stepConfig.Timeout, "timeout", `600`, "timeout for http layer in seconds") | ||||
| @@ -344,15 +342,6 @@ func malwareExecuteScanMetadata() config.StepData { | ||||
| 						Aliases:   []config.Alias{}, | ||||
| 						Default:   os.Getenv("PIPER_scanImage"), | ||||
| 					}, | ||||
| 					{ | ||||
| 						Name:        "scanImageIncludeLayers", | ||||
| 						ResourceRef: []config.ResourceReference{}, | ||||
| 						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"}, | ||||
| 						Type:        "bool", | ||||
| 						Mandatory:   false, | ||||
| 						Aliases:     []config.Alias{}, | ||||
| 						Default:     true, | ||||
| 					}, | ||||
| 					{ | ||||
| 						Name: "scanImageRegistryUrl", | ||||
| 						ResourceRef: []config.ResourceReference{ | ||||
|   | ||||
| @@ -9,7 +9,9 @@ import ( | ||||
| 	"strings" | ||||
| 	"testing" | ||||
|  | ||||
| 	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util" | ||||
| 	"github.com/google/go-containerregistry/pkg/v1" | ||||
| 	"github.com/google/go-containerregistry/pkg/v1/fake" | ||||
|  | ||||
| 	piperDocker "github.com/SAP/jenkins-library/pkg/docker" | ||||
| 	piperhttp "github.com/SAP/jenkins-library/pkg/http" | ||||
| 	"github.com/SAP/jenkins-library/pkg/malwarescan" | ||||
| @@ -57,7 +59,7 @@ func (utils *malwareScanUtilsMockBundle) Scan(candidate io.Reader) (*malwarescan | ||||
| } | ||||
|  | ||||
| func (utils *malwareScanUtilsMockBundle) newDockerClient(options piperDocker.ClientOptions) piperDocker.Download { | ||||
| 	return &dockerClientMock{imageName: options.ImageName, registryURL: options.RegistryURL, localPath: options.LocalPath, includeLayers: options.IncludeLayers} | ||||
| 	return &dockerClientMock{imageName: options.ImageName, registryURL: options.RegistryURL, localPath: options.LocalPath} | ||||
| } | ||||
|  | ||||
| func TestMalwareScanWithNoBuildtool(t *testing.T) { | ||||
| @@ -287,16 +289,12 @@ type dockerClientMock struct { | ||||
| 	includeLayers bool | ||||
| } | ||||
|  | ||||
| func (c *dockerClientMock) GetImageSource() (string, error) { | ||||
| 	return c.imageName, nil | ||||
| //DownloadImage download the image to the specified path | ||||
| func (c *dockerClientMock) DownloadImage(imageSource, filePath string) (v1.Image, error) { | ||||
| 	return &fake.FakeImage{}, nil // fmt.Errorf("%s", filePath) | ||||
| } | ||||
|  | ||||
| //DownloadImageToPath download the image to the specified path | ||||
| func (c *dockerClientMock) DownloadImageToPath(imageSource, filePath string) (pkgutil.Image, error) { | ||||
| 	return pkgutil.Image{}, nil // fmt.Errorf("%s", filePath) | ||||
| } | ||||
|  | ||||
| //TarImage write a tar from the given image | ||||
| func (c *dockerClientMock) TarImage(writer io.Writer, image pkgutil.Image) error { | ||||
| 	return nil | ||||
| //DownloadImage download the image to the specified path | ||||
| func (c *dockerClientMock) DownloadImageContent(imageSource, filePath string) (v1.Image, error) { | ||||
| 	return &fake.FakeImage{}, nil // fmt.Errorf("%s", filePath) | ||||
| } | ||||
|   | ||||
| @@ -4,7 +4,6 @@ import ( | ||||
| 	"encoding/json" | ||||
| 	"fmt" | ||||
| 	"io" | ||||
| 	"io/ioutil" | ||||
| 	"os" | ||||
| 	"path" | ||||
| 	"path/filepath" | ||||
| @@ -15,11 +14,10 @@ import ( | ||||
|  | ||||
| 	"github.com/pkg/errors" | ||||
|  | ||||
| 	"github.com/GoogleContainerTools/container-diff/pkg/util" | ||||
| 	"github.com/SAP/jenkins-library/pkg/command" | ||||
| 	piperDocker "github.com/SAP/jenkins-library/pkg/docker" | ||||
| 	"github.com/SAP/jenkins-library/pkg/log" | ||||
| 	StepResults "github.com/SAP/jenkins-library/pkg/piperutils" | ||||
| 	"github.com/SAP/jenkins-library/pkg/piperutils" | ||||
| 	"github.com/SAP/jenkins-library/pkg/protecode" | ||||
| 	"github.com/SAP/jenkins-library/pkg/telemetry" | ||||
| 	"github.com/SAP/jenkins-library/pkg/toolrecord" | ||||
| @@ -32,10 +30,15 @@ const ( | ||||
| 	stepResultFile = "protecodeExecuteScan.json" | ||||
| ) | ||||
|  | ||||
| var reportPath = "./" | ||||
| var cachePath = "./cache" | ||||
| var cacheProtecodeImagePath = "/protecode/Image" | ||||
| var cacheProtecodePath = "/protecode" | ||||
| type protecodeUtils interface { | ||||
| 	piperutils.FileUtils | ||||
| 	piperDocker.Download | ||||
| } | ||||
|  | ||||
| type protecodeUtilsBundle struct { | ||||
| 	*piperutils.Files | ||||
| 	*piperDocker.Client | ||||
| } | ||||
|  | ||||
| func protecodeExecuteScan(config protecodeExecuteScanOptions, telemetryData *telemetry.CustomData, influx *protecodeExecuteScanInflux) { | ||||
| 	c := command.Command{} | ||||
| @@ -43,24 +46,40 @@ func protecodeExecuteScan(config protecodeExecuteScanOptions, telemetryData *tel | ||||
| 	c.Stdout(log.Writer()) | ||||
| 	c.Stderr(log.Writer()) | ||||
|  | ||||
| 	dClient := createDockerClient(&config) | ||||
| 	//create client for sending api request | ||||
| 	log.Entry().Debug("Create protecode client") | ||||
| 	client := createProtecodeClient(&config) | ||||
|  | ||||
| 	dClientOptions := piperDocker.ClientOptions{ImageName: config.ScanImage, RegistryURL: config.DockerRegistryURL, LocalPath: config.FilePath} | ||||
| 	dClient := &piperDocker.Client{} | ||||
| 	dClient.SetOptions(dClientOptions) | ||||
|  | ||||
| 	utils := protecodeUtilsBundle{ | ||||
| 		Client: dClient, | ||||
| 		Files:  &piperutils.Files{}, | ||||
| 	} | ||||
|  | ||||
| 	influx.step_data.fields.protecode = false | ||||
| 	if err := runProtecodeScan(&config, influx, dClient); err != nil { | ||||
| 	if err := runProtecodeScan(&config, influx, client, utils, "./cache"); err != nil { | ||||
| 		log.Entry().WithError(err).Fatal("Failed to execute protecode scan.") | ||||
| 	} | ||||
| 	influx.step_data.fields.protecode = true | ||||
| } | ||||
|  | ||||
| func runProtecodeScan(config *protecodeExecuteScanOptions, influx *protecodeExecuteScanInflux, dClient piperDocker.Download) error { | ||||
| func runProtecodeScan(config *protecodeExecuteScanOptions, influx *protecodeExecuteScanInflux, client protecode.Protecode, utils protecodeUtils, cachePath string) error { | ||||
| 	// make sure cache exists | ||||
| 	if err := utils.MkdirAll(cachePath, 0755); err != nil { | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| 	correctDockerConfigEnvVar(config) | ||||
|  | ||||
| 	var fileName, filePath string | ||||
| 	var err error | ||||
| 	//create client for sending api request | ||||
| 	log.Entry().Debug("Create protecode client") | ||||
| 	client := createClient(config) | ||||
|  | ||||
| 	if len(config.FetchURL) == 0 && len(config.FilePath) == 0 { | ||||
| 		log.Entry().Debugf("Get docker image: %v, %v, %v, %v", config.ScanImage, config.DockerRegistryURL, config.FilePath, config.IncludeLayers) | ||||
| 		fileName, filePath, err = getDockerImage(dClient, config) | ||||
| 		log.Entry().Debugf("Get docker image: %v, %v, %v", config.ScanImage, config.DockerRegistryURL, config.FilePath) | ||||
| 		fileName, filePath, err = getDockerImage(utils, config, cachePath) | ||||
| 		if err != nil { | ||||
| 			return errors.Wrap(err, "failed to get Docker image") | ||||
| 		} | ||||
| @@ -85,13 +104,13 @@ func runProtecodeScan(config *protecodeExecuteScanOptions, influx *protecodeExec | ||||
| 	} | ||||
|  | ||||
| 	log.Entry().Debug("Execute protecode scan") | ||||
| 	if err := executeProtecodeScan(influx, client, config, fileName, writeReportToFile); err != nil { | ||||
| 	if err := executeProtecodeScan(influx, client, config, fileName, utils); err != nil { | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| 	defer os.Remove(config.FilePath) | ||||
| 	defer utils.FileRemove(config.FilePath) | ||||
|  | ||||
| 	if err := os.RemoveAll(filepath.Join(cachePath, cacheProtecodePath)); err != nil { | ||||
| 	if err := utils.RemoveAll(cachePath); err != nil { | ||||
| 		log.Entry().Warnf("Error during cleanup folder %v", err) | ||||
| 	} | ||||
|  | ||||
| @@ -109,55 +128,25 @@ func handleArtifactVersion(artifactVersion string) string { | ||||
| 	return artifactVersion | ||||
| } | ||||
|  | ||||
| func getDockerImage(dClient piperDocker.Download, config *protecodeExecuteScanOptions) (string, string, error) { | ||||
| func getDockerImage(utils protecodeUtils, config *protecodeExecuteScanOptions, cachePath string) (string, string, error) { | ||||
| 	m := regexp.MustCompile("[\\s@:/]") | ||||
|  | ||||
| 	cacheImagePath := filepath.Join(cachePath, cacheProtecodeImagePath) | ||||
| 	deletePath := filepath.Join(cachePath, cacheProtecodePath) | ||||
| 	err := os.RemoveAll(deletePath) | ||||
| 	tarFileName := fmt.Sprintf("%s.tar", m.ReplaceAllString(config.ScanImage, "-")) | ||||
| 	tarFilePath, err := filepath.Abs(filepath.Join(cachePath, tarFileName)) | ||||
|  | ||||
| 	os.Mkdir(cacheImagePath, 600) | ||||
|  | ||||
| 	imageSource, err := dClient.GetImageSource() | ||||
| 	if err != nil { | ||||
| 		log.SetErrorCategory(log.ErrorConfiguration) | ||||
| 		return "", "", errors.Wrap(err, "failed to get docker image") | ||||
| 		return "", "", err | ||||
| 	} | ||||
| 	image, err := dClient.DownloadImageToPath(imageSource, cacheImagePath) | ||||
| 	if err != nil { | ||||
|  | ||||
| 	if _, err = utils.DownloadImage(config.ScanImage, tarFilePath); err != nil { | ||||
| 		return "", "", errors.Wrap(err, "failed to download docker image") | ||||
| 	} | ||||
|  | ||||
| 	var fileName string | ||||
| 	if util.IsTar(config.ScanImage) { | ||||
| 		fileName = config.ScanImage | ||||
| 	} else { | ||||
| 		fileName = getTarName(config) | ||||
| 		tarFilePath := filepath.Join(cachePath, fileName) | ||||
| 		tarFile, err := os.Create(tarFilePath) | ||||
| 		if err != nil { | ||||
| 			log.SetErrorCategory(log.ErrorCustom) | ||||
| 			return "", "", errors.Wrap(err, "failed to create tar for the docker image") | ||||
| 		} | ||||
| 		defer tarFile.Close() | ||||
| 		if err := os.Chmod(tarFilePath, 0644); err != nil { | ||||
| 			log.SetErrorCategory(log.ErrorCustom) | ||||
| 			return "", "", errors.Wrap(err, "failed to set permissions on tar for the docker image") | ||||
| 		} | ||||
| 		if err = dClient.TarImage(tarFile, image); err != nil { | ||||
| 			return "", "", errors.Wrap(err, "failed to tar the docker image") | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	resultFilePath := config.FilePath | ||||
|  | ||||
| 	if len(config.FilePath) <= 0 { | ||||
| 		resultFilePath = cachePath | ||||
| 	} | ||||
|  | ||||
| 	return fileName, resultFilePath, nil | ||||
| 	return filepath.Base(tarFilePath), filepath.Dir(tarFilePath), nil | ||||
| } | ||||
|  | ||||
| func executeProtecodeScan(influx *protecodeExecuteScanInflux, client protecode.Protecode, config *protecodeExecuteScanOptions, fileName string, writeReportToFile func(resp io.ReadCloser, reportFileName string) error) error { | ||||
| func executeProtecodeScan(influx *protecodeExecuteScanInflux, client protecode.Protecode, config *protecodeExecuteScanOptions, fileName string, utils protecodeUtils) error { | ||||
| 	reportPath := "./" | ||||
|  | ||||
| 	log.Entry().Debugf("[DEBUG] ===> Load existing product Group:%v, VerifyOnly:%v, Filename:%v, replaceProductId:%v", config.Group, config.VerifyOnly, fileName, config.ReplaceProductID) | ||||
|  | ||||
| @@ -194,7 +183,7 @@ func executeProtecodeScan(influx *protecodeExecuteScanInflux, client protecode.P | ||||
| 	} | ||||
|  | ||||
| 	// check if no existing is found | ||||
| 	productID = uploadScanOrDeclareFetch(*config, productID, client, fileName) | ||||
| 	productID = uploadScanOrDeclareFetch(utils, *config, productID, client, fileName) | ||||
|  | ||||
| 	log.Entry().Debugf("[DEBUG] ===> After 'uploadScanOrDeclareFetch' returned productID: %v", productID) | ||||
|  | ||||
| @@ -207,7 +196,7 @@ func executeProtecodeScan(influx *protecodeExecuteScanInflux, client protecode.P | ||||
| 	result := client.PollForResult(productID, config.TimeoutMinutes) | ||||
| 	// write results to file | ||||
| 	jsonData, _ := json.Marshal(result) | ||||
| 	ioutil.WriteFile(filepath.Join(reportPath, scanResultFile), jsonData, 0644) | ||||
| 	utils.FileWrite(filepath.Join(reportPath, scanResultFile), jsonData, 0644) | ||||
|  | ||||
| 	//check if result is ok else notify | ||||
| 	if protecode.HasFailed(result) { | ||||
| @@ -218,10 +207,17 @@ func executeProtecodeScan(influx *protecodeExecuteScanInflux, client protecode.P | ||||
| 	//loadReport | ||||
| 	log.Entry().Debugf("Load report %v for %v", config.ReportFileName, productID) | ||||
| 	resp := client.LoadReport(config.ReportFileName, productID) | ||||
| 	//save report to filesystem | ||||
| 	if err := writeReportToFile(*resp, config.ReportFileName); err != nil { | ||||
|  | ||||
| 	buf, err := io.ReadAll(*resp) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		return fmt.Errorf("unable to process protecode report %v", err) | ||||
| 	} | ||||
|  | ||||
| 	if err = utils.FileWrite(config.ReportFileName, buf, 0644); err != nil { | ||||
| 		log.Entry().Warningf("failed to write report: %s", err) | ||||
| 	} | ||||
|  | ||||
| 	//clean scan from server | ||||
| 	log.Entry().Debugf("Delete scan %v for %v", config.CleanupMode, productID) | ||||
| 	client.DeleteScan(config.CleanupMode, productID) | ||||
| @@ -239,7 +235,7 @@ func executeProtecodeScan(influx *protecodeExecuteScanInflux, client protecode.P | ||||
| 			Target:                      config.ReportFileName, | ||||
| 			Vulnerabilities:             vulns, | ||||
| 			ProductID:                   fmt.Sprintf("%v", productID), | ||||
| 		}, reportPath, stepResultFile, parsedResult, ioutil.WriteFile); err != nil { | ||||
| 		}, reportPath, stepResultFile, parsedResult, utils); err != nil { | ||||
| 		log.Entry().Warningf("failed to write report: %v", err) | ||||
| 	} | ||||
|  | ||||
| @@ -247,21 +243,21 @@ func executeProtecodeScan(influx *protecodeExecuteScanInflux, client protecode.P | ||||
| 	setInfluxData(influx, parsedResult) | ||||
|  | ||||
| 	// write reports JSON | ||||
| 	reports := []StepResults.Path{ | ||||
| 	reports := []piperutils.Path{ | ||||
| 		{Target: config.ReportFileName, Mandatory: true}, | ||||
| 		{Target: stepResultFile, Mandatory: true}, | ||||
| 		{Target: scanResultFile, Mandatory: true}, | ||||
| 	} | ||||
| 	// write links JSON | ||||
| 	webuiURL := fmt.Sprintf(webReportPath, config.ServerURL, productID) | ||||
| 	links := []StepResults.Path{ | ||||
| 	links := []piperutils.Path{ | ||||
| 		{Name: "Protecode WebUI", Target: webuiURL}, | ||||
| 		{Name: "Protecode Report", Target: path.Join("artifact", config.ReportFileName), Scope: "job"}, | ||||
| 	} | ||||
|  | ||||
| 	// write custom report | ||||
| 	scanReport := protecode.CreateCustomReport(fileName, productID, parsedResult, vulns) | ||||
| 	paths, err := protecode.WriteCustomReports(scanReport, fileName, fmt.Sprint(productID)) | ||||
| 	paths, err := protecode.WriteCustomReports(scanReport, fileName, fmt.Sprint(productID), utils) | ||||
| 	if err != nil { | ||||
| 		// do not fail - consider failing later on | ||||
| 		log.Entry().Warning("failed to create custom HTML/MarkDown file ...", err) | ||||
| @@ -275,10 +271,10 @@ func executeProtecodeScan(influx *protecodeExecuteScanInflux, client protecode.P | ||||
| 		// do not fail until the framework is well established | ||||
| 		log.Entry().Warning("TR_PROTECODE: Failed to create toolrecord file ...", err) | ||||
| 	} else { | ||||
| 		reports = append(reports, StepResults.Path{Target: toolRecordFileName}) | ||||
| 		reports = append(reports, piperutils.Path{Target: toolRecordFileName}) | ||||
| 	} | ||||
|  | ||||
| 	StepResults.PersistReportsAndLinks("protecodeExecuteScan", "", reports, links) | ||||
| 	piperutils.PersistReportsAndLinks("protecodeExecuteScan", "", reports, links) | ||||
|  | ||||
| 	if config.FailOnSevereVulnerabilities && protecode.HasSevereVulnerabilities(result.Result, config.ExcludeCVEs) { | ||||
| 		log.SetErrorCategory(log.ErrorCompliance) | ||||
| @@ -296,8 +292,7 @@ func setInfluxData(influx *protecodeExecuteScanInflux, result map[string]int) { | ||||
| 	influx.protecode_data.fields.vulnerabilities = result["vulnerabilities"] | ||||
| } | ||||
|  | ||||
| func createClient(config *protecodeExecuteScanOptions) protecode.Protecode { | ||||
|  | ||||
| func createProtecodeClient(config *protecodeExecuteScanOptions) protecode.Protecode { | ||||
| 	var duration time.Duration = time.Duration(time.Minute * 1) | ||||
|  | ||||
| 	if len(config.TimeoutMinutes) > 0 { | ||||
| @@ -324,16 +319,7 @@ func createClient(config *protecodeExecuteScanOptions) protecode.Protecode { | ||||
| 	return pc | ||||
| } | ||||
|  | ||||
| func createDockerClient(config *protecodeExecuteScanOptions) piperDocker.Download { | ||||
|  | ||||
| 	dClientOptions := piperDocker.ClientOptions{ImageName: config.ScanImage, RegistryURL: config.DockerRegistryURL, LocalPath: config.FilePath, IncludeLayers: config.IncludeLayers} | ||||
| 	dClient := &piperDocker.Client{} | ||||
| 	dClient.SetOptions(dClientOptions) | ||||
|  | ||||
| 	return dClient | ||||
| } | ||||
|  | ||||
| func uploadScanOrDeclareFetch(config protecodeExecuteScanOptions, productID int, client protecode.Protecode, fileName string) int { | ||||
| func uploadScanOrDeclareFetch(utils protecodeUtils, config protecodeExecuteScanOptions, productID int, client protecode.Protecode, fileName string) int { | ||||
| 	//check if the LoadExistingProduct) before returns an valid product id, than skip this | ||||
| 	//if !hasExisting(productID, config.VerifyOnly) { | ||||
|  | ||||
| @@ -343,7 +329,7 @@ func uploadScanOrDeclareFetch(config protecodeExecuteScanOptions, productID int, | ||||
| 	if productID <= 0 { | ||||
| 		log.Entry().Infof("New product creation started ... ") | ||||
| 		// log.Entry().Debugf("[DEBUG] ===> New product creation started: %v", productID) | ||||
| 		productID = uploadFile(config, productID, client, fileName, false) | ||||
| 		productID = uploadFile(utils, config, productID, client, fileName, false) | ||||
|  | ||||
| 		log.Entry().Infof("New product has been successfully created: %v", productID) | ||||
| 		// log.Entry().Debugf("[DEBUG] ===> After uploading [productID < 0] file returned productID: %v", productID) | ||||
| @@ -353,7 +339,7 @@ func uploadScanOrDeclareFetch(config protecodeExecuteScanOptions, productID int, | ||||
| 	} else if (productID > 0) && !config.VerifyOnly { | ||||
| 		log.Entry().Infof("Product already exists and 'VerifyOnly (reuseExisting)' is false then product (%v) binary and scan result will be replaced without creating a new product.", productID) | ||||
| 		// log.Entry().Debugf("[DEBUG] ===> Replace binary entry point started %v", productID) | ||||
| 		productID = uploadFile(config, productID, client, fileName, true) | ||||
| 		productID = uploadFile(utils, config, productID, client, fileName, true) | ||||
|  | ||||
| 		// log.Entry().Debugf("[DEBUG] ===> After uploading file [(productID > 0) && !config.VerifyOnly] returned productID: %v", productID) | ||||
| 		return productID | ||||
| @@ -366,7 +352,7 @@ func uploadScanOrDeclareFetch(config protecodeExecuteScanOptions, productID int, | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func uploadFile(config protecodeExecuteScanOptions, productID int, client protecode.Protecode, fileName string, replaceBinary bool) int { | ||||
| func uploadFile(utils protecodeUtils, config protecodeExecuteScanOptions, productID int, client protecode.Protecode, fileName string, replaceBinary bool) int { | ||||
|  | ||||
| 	// get calculated version for Version field | ||||
| 	version := getProcessedVersion(&config) | ||||
| @@ -381,7 +367,7 @@ func uploadFile(config protecodeExecuteScanOptions, productID int, client protec | ||||
| 			log.Entry().Fatalf("There is no file path configured for upload : %v", config.FilePath) | ||||
| 		} | ||||
| 		pathToFile := filepath.Join(config.FilePath, fileName) | ||||
| 		if !(fileExists(pathToFile)) { | ||||
| 		if exists, err := utils.FileExists(pathToFile); err != nil && !exists { | ||||
| 			log.Entry().Fatalf("There is no file for upload: %v", pathToFile) | ||||
| 		} | ||||
|  | ||||
| @@ -397,14 +383,6 @@ func uploadFile(config protecodeExecuteScanOptions, productID int, client protec | ||||
| 	return productID | ||||
| } | ||||
|  | ||||
| func fileExists(filename string) bool { | ||||
| 	info, err := os.Stat(filename) | ||||
| 	if os.IsNotExist(err) { | ||||
| 		return false | ||||
| 	} | ||||
| 	return !info.IsDir() | ||||
| } | ||||
|  | ||||
| func hasExisting(productID int, verifyOnly bool) bool { | ||||
| 	if (productID > 0) || verifyOnly { | ||||
| 		return true | ||||
| @@ -412,17 +390,6 @@ func hasExisting(productID int, verifyOnly bool) bool { | ||||
| 	return false | ||||
| } | ||||
|  | ||||
| var writeReportToFile = func(resp io.ReadCloser, reportFileName string) error { | ||||
| 	filePath := filepath.Join(reportPath, reportFileName) | ||||
| 	f, err := os.Create(filePath) | ||||
| 	if err == nil { | ||||
| 		defer f.Close() | ||||
| 		_, err = io.Copy(f, resp) | ||||
| 	} | ||||
|  | ||||
| 	return err | ||||
| } | ||||
|  | ||||
| func correctDockerConfigEnvVar(config *protecodeExecuteScanOptions) { | ||||
| 	path := config.DockerConfigJSON | ||||
| 	if len(path) > 0 { | ||||
| @@ -436,26 +403,6 @@ func correctDockerConfigEnvVar(config *protecodeExecuteScanOptions) { | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func getTarName(config *protecodeExecuteScanOptions) string { | ||||
|  | ||||
| 	// remove original version | ||||
| 	fileName := strings.TrimSuffix(config.ScanImage, ":"+config.Version) | ||||
| 	// remove sha digest if exists | ||||
| 	sha256 := "@sha256" | ||||
| 	if index := strings.Index(fileName, sha256); index > -1 { | ||||
| 		fileName = fileName[:index] | ||||
| 	} | ||||
|  | ||||
| 	version := getProcessedVersion(config) | ||||
| 	if len(version) > 0 { | ||||
| 		fileName = fileName + "_" + version | ||||
| 	} | ||||
|  | ||||
| 	fileName = strings.ReplaceAll(fileName, "/", "_") | ||||
|  | ||||
| 	return fileName + ".tar" | ||||
| } | ||||
|  | ||||
| // Calculate version based on versioning model and artifact version or return custom scan version provided by user | ||||
| func getProcessedVersion(config *protecodeExecuteScanOptions) string { | ||||
| 	processedVersion := config.CustomScanVersion | ||||
|   | ||||
| @@ -29,7 +29,6 @@ type protecodeExecuteScanOptions struct { | ||||
| 	DockerConfigJSON            string `json:"dockerConfigJSON,omitempty"` | ||||
| 	CleanupMode                 string `json:"cleanupMode,omitempty" validate:"possible-values=none binary complete"` | ||||
| 	FilePath                    string `json:"filePath,omitempty"` | ||||
| 	IncludeLayers               bool   `json:"includeLayers,omitempty"` | ||||
| 	TimeoutMinutes              string `json:"timeoutMinutes,omitempty"` | ||||
| 	ServerURL                   string `json:"serverUrl,omitempty"` | ||||
| 	ReportFileName              string `json:"reportFileName,omitempty"` | ||||
| @@ -242,7 +241,6 @@ func addProtecodeExecuteScanFlags(cmd *cobra.Command, stepConfig *protecodeExecu | ||||
| 	cmd.Flags().StringVar(&stepConfig.DockerConfigJSON, "dockerConfigJSON", os.Getenv("PIPER_dockerConfigJSON"), "Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).") | ||||
| 	cmd.Flags().StringVar(&stepConfig.CleanupMode, "cleanupMode", `binary`, "Decides which parts are removed from the Protecode backend after the scan") | ||||
| 	cmd.Flags().StringVar(&stepConfig.FilePath, "filePath", os.Getenv("PIPER_filePath"), "The path to the file from local workspace to scan with Protecode") | ||||
| 	cmd.Flags().BoolVar(&stepConfig.IncludeLayers, "includeLayers", false, "Flag if the docker layers should be included") | ||||
| 	cmd.Flags().StringVar(&stepConfig.TimeoutMinutes, "timeoutMinutes", `60`, "The timeout to wait for the scan to finish") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ServerURL, "serverUrl", os.Getenv("PIPER_serverUrl"), "The URL to the Protecode backend") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ReportFileName, "reportFileName", `protecode_report.pdf`, "The file name of the report to be created") | ||||
| @@ -367,15 +365,6 @@ func protecodeExecuteScanMetadata() config.StepData { | ||||
| 						Aliases:     []config.Alias{}, | ||||
| 						Default:     os.Getenv("PIPER_filePath"), | ||||
| 					}, | ||||
| 					{ | ||||
| 						Name:        "includeLayers", | ||||
| 						ResourceRef: []config.ResourceReference{}, | ||||
| 						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"}, | ||||
| 						Type:        "bool", | ||||
| 						Mandatory:   false, | ||||
| 						Aliases:     []config.Alias{}, | ||||
| 						Default:     false, | ||||
| 					}, | ||||
| 					{ | ||||
| 						Name:        "timeoutMinutes", | ||||
| 						ResourceRef: []config.ResourceReference{}, | ||||
|   | ||||
| @@ -6,95 +6,32 @@ import ( | ||||
| 	"bytes" | ||||
| 	"encoding/json" | ||||
| 	"fmt" | ||||
| 	"io" | ||||
| 	"io/ioutil" | ||||
| 	"net/http" | ||||
| 	"net/http/httptest" | ||||
| 	"net/url" | ||||
| 	"os" | ||||
| 	"path/filepath" | ||||
| 	"strings" | ||||
| 	"time" | ||||
|  | ||||
| 	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util" | ||||
| 	piperHttp "github.com/SAP/jenkins-library/pkg/http" | ||||
| 	"github.com/SAP/jenkins-library/pkg/mock" | ||||
| 	"github.com/SAP/jenkins-library/pkg/protecode" | ||||
|  | ||||
| 	"github.com/stretchr/testify/assert" | ||||
| 	"github.com/stretchr/testify/require" | ||||
|  | ||||
| 	"github.com/google/go-containerregistry/pkg/v1" | ||||
| 	"github.com/google/go-containerregistry/pkg/v1/fake" | ||||
| ) | ||||
|  | ||||
| type DockerClientMock struct { | ||||
| 	imageName     string | ||||
| 	registryURL   string | ||||
| 	localPath     string | ||||
| 	includeLayers bool | ||||
| } | ||||
|  | ||||
| //Download interface for download an image to a local path | ||||
| type Download interface { | ||||
| 	GetImageSource() (string, error) | ||||
| 	DownloadImageToPath(imageSource, filePath string) (pkgutil.Image, error) | ||||
| 	TarImage(writer io.Writer, image pkgutil.Image) error | ||||
| } | ||||
|  | ||||
| const ( | ||||
| 	daemonPrefix = "daemon://" | ||||
| 	remotePrefix = "remote://" | ||||
| ) | ||||
|  | ||||
| func (c *DockerClientMock) GetImageSource() (string, error) { | ||||
|  | ||||
| 	imageSource := c.imageName | ||||
|  | ||||
| 	if len(c.registryURL) > 0 && len(c.localPath) <= 0 { | ||||
| 		registry := c.registryURL | ||||
|  | ||||
| 		url, _ := url.Parse(c.registryURL) | ||||
| 		//remove protocoll from registryURL to get registry | ||||
| 		if len(url.Scheme) > 0 { | ||||
| 			registry = strings.Replace(c.registryURL, fmt.Sprintf("%v://", url.Scheme), "", 1) | ||||
| 		} | ||||
|  | ||||
| 		if strings.HasSuffix(registry, "/") { | ||||
| 			imageSource = fmt.Sprintf("%v%v%v", remotePrefix, registry, c.imageName) | ||||
| 		} else { | ||||
| 			imageSource = fmt.Sprintf("%v%v/%v", remotePrefix, registry, c.imageName) | ||||
| 		} | ||||
| 	} else if len(c.localPath) > 0 { | ||||
| 		imageSource = c.localPath | ||||
| 		if !pkgutil.IsTar(c.localPath) { | ||||
| 			imageSource = fmt.Sprintf("%v%v", daemonPrefix, c.localPath) | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	if len(imageSource) <= 0 { | ||||
| 		return imageSource, fmt.Errorf("There is no image source for the parameters: (Name: %v, Registry: %v, local Path: %v)", c.imageName, c.registryURL, c.localPath) | ||||
| 	} | ||||
|  | ||||
| 	return imageSource, nil | ||||
| } | ||||
|  | ||||
| //DownloadImageToPath download the image to the specified path | ||||
| func (c *DockerClientMock) DownloadImageToPath(imageSource, filePath string) (pkgutil.Image, error) { | ||||
|  | ||||
| 	return pkgutil.Image{}, nil | ||||
| } | ||||
|  | ||||
| //TarImage write a tar from the given image | ||||
| func (c *DockerClientMock) TarImage(writer io.Writer, image pkgutil.Image) error { | ||||
|  | ||||
| 	return nil | ||||
| type protecodeTestUtilsBundle struct { | ||||
| 	*mock.FilesMock | ||||
| 	*mock.DownloadMock | ||||
| } | ||||
|  | ||||
| func TestRunProtecodeScan(t *testing.T) { | ||||
| 	requestURI := "" | ||||
| 	dir, err := ioutil.TempDir("", "t") | ||||
| 	require.NoError(t, err, "Failed to create temporary directory") | ||||
| 	// clean up tmp dir | ||||
| 	defer func() { _ = os.RemoveAll(dir) }() | ||||
| 	testFile, err := ioutil.TempFile(dir, "t.tar") | ||||
| 	require.NoError(t, err) | ||||
| 	fileName := filepath.Base(testFile.Name()) | ||||
| 	path := strings.ReplaceAll(testFile.Name(), fileName, "") | ||||
|  | ||||
| 	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { | ||||
| 		requestURI = req.RequestURI | ||||
| @@ -141,24 +78,81 @@ func TestRunProtecodeScan(t *testing.T) { | ||||
| 	pc := protecode.Protecode{} | ||||
| 	pc.SetOptions(po) | ||||
|  | ||||
| 	dClient := &DockerClientMock{imageName: "t", registryURL: "", localPath: path, includeLayers: false} | ||||
|  | ||||
| 	influx := protecodeExecuteScanInflux{} | ||||
| 	reportPath = dir | ||||
| 	cachePath = dir | ||||
|  | ||||
| 	t.Run("With tar as scan image", func(t *testing.T) { | ||||
| 		config := protecodeExecuteScanOptions{ServerURL: server.URL, TimeoutMinutes: "1", VerifyOnly: false, CleanupMode: "none", Group: "13", FetchURL: "/api/fetch/", ExcludeCVEs: "CVE-2018-1, CVE-2017-1000382", ReportFileName: "./cache/report-file.txt"} | ||||
| 		err = runProtecodeScan(&config, &influx, dClient) | ||||
| 		assert.NoError(t, err) | ||||
| 	}) | ||||
| 	ttt := []struct { | ||||
| 		name string | ||||
| 		opts protecodeExecuteScanOptions | ||||
| 	}{ | ||||
| 		{ | ||||
| 			name: "With tar as scan image", | ||||
| 			opts: protecodeExecuteScanOptions{ | ||||
| 				ServerURL:      server.URL, | ||||
| 				TimeoutMinutes: "1", | ||||
| 				VerifyOnly:     false, | ||||
| 				CleanupMode:    "none", | ||||
| 				Group:          "13", | ||||
| 				FetchURL:       "/api/fetch/", | ||||
| 				ExcludeCVEs:    "CVE-2018-1, CVE-2017-1000382", | ||||
| 				ReportFileName: "./cache/report-file.txt", | ||||
| 			}, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name: "Without tar as scan image", | ||||
| 			opts: protecodeExecuteScanOptions{ | ||||
| 				ServerURL:      server.URL, | ||||
| 				ScanImage:      "t", | ||||
| 				TimeoutMinutes: "1", | ||||
| 				VerifyOnly:     false, | ||||
| 				CleanupMode:    "none", | ||||
| 				Group:          "13", | ||||
| 				ExcludeCVEs:    "CVE-2018-1, CVE-2017-1000382", | ||||
| 				ReportFileName: "./cache/report-file.txt", | ||||
| 			}, | ||||
| 		}, | ||||
| 	} | ||||
|  | ||||
| 	t.Run("Without tar as scan image", func(t *testing.T) { | ||||
| 		config := protecodeExecuteScanOptions{ServerURL: server.URL, ScanImage: "t", TimeoutMinutes: "1", VerifyOnly: false, CleanupMode: "none", Group: "13", ExcludeCVEs: "CVE-2018-1, CVE-2017-1000382", ReportFileName: "./cache/report-file.txt"} | ||||
| 		err = runProtecodeScan(&config, &influx, dClient) | ||||
| 		assert.NoError(t, err) | ||||
| 	}) | ||||
| 	for _, test := range ttt { | ||||
| 		t.Run(test.name, func(t *testing.T) { | ||||
| 			files := mock.FilesMock{} | ||||
|  | ||||
| 			httpClient := piperHttp.Client{} | ||||
| 			httpClient.SetFileUtils(&files) | ||||
|  | ||||
| 			pc.SetHttpClient(&httpClient) | ||||
|  | ||||
| 			docker := mock.DownloadMock{} | ||||
| 			docker.Stub = func(imageRef string, dest string) (v1.Image, error) { | ||||
| 				files.AddFile(dest, []byte("")) | ||||
| 				return &fake.FakeImage{}, nil | ||||
| 			} | ||||
|  | ||||
| 			utils := protecodeTestUtilsBundle{DownloadMock: &docker, FilesMock: &files} | ||||
|  | ||||
| 			cacheDir, _ := files.TempDir("", "protecode-") | ||||
| 			files.AddFile(filepath.Join(cacheDir, "t.tar"), []byte("")) | ||||
|  | ||||
| 			err := runProtecodeScan(&test.opts, &influx, pc, utils, cacheDir) | ||||
|  | ||||
| 			if assert.NoError(t, err) { | ||||
| 				if protecodeExecuteJsonExists, err := files.FileExists("protecodeExecuteScan.json"); assert.NoError(t, err) { | ||||
| 					assert.True(t, protecodeExecuteJsonExists, "protecodeExecuteScan.json expected") | ||||
| 				} | ||||
|  | ||||
| 				if protecodeVulnsJsonExists, err := files.FileExists("protecodescan_vulns.json"); assert.NoError(t, err) { | ||||
| 					assert.True(t, protecodeVulnsJsonExists, "protecodescan_vulns.json expected") | ||||
| 				} | ||||
|  | ||||
| 				if userSpecifiedReportExists, err := files.FileExists(test.opts.ReportFileName); assert.NoError(t, err) { | ||||
| 					assert.True(t, userSpecifiedReportExists, "%s must exist", test.opts.ReportFileName) | ||||
| 				} | ||||
| 			} | ||||
|  | ||||
| 			if cacheExists, err := files.DirExists(cacheDir); assert.NoError(t, err) { | ||||
| 				assert.True(t, cacheExists, "Whoever calls runProtecodeScan is responsible to cleanup the cache.") | ||||
| 			} | ||||
| 		}) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestHandleArtifactVersion(t *testing.T) { | ||||
| @@ -175,13 +169,15 @@ func TestHandleArtifactVersion(t *testing.T) { | ||||
| 		{"7.20.20", "7.20.20"}, | ||||
| 	} | ||||
|  | ||||
| 	for _, c := range cases { | ||||
| 		got := handleArtifactVersion(c.version) | ||||
| 		assert.Equal(t, c.want, got) | ||||
| 	for i, c := range cases { | ||||
| 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { | ||||
| 			got := handleArtifactVersion(c.version) | ||||
| 			assert.Equal(t, c.want, got) | ||||
| 		}) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestCreateClient(t *testing.T) { | ||||
| func TestCreateProtecodeClient(t *testing.T) { | ||||
| 	cases := []struct { | ||||
| 		timeout string | ||||
| 	}{ | ||||
| @@ -190,28 +186,12 @@ func TestCreateClient(t *testing.T) { | ||||
| 	} | ||||
|  | ||||
| 	for _, c := range cases { | ||||
| 		config := protecodeExecuteScanOptions{TimeoutMinutes: c.timeout} | ||||
| 		t.Run(fmt.Sprintf("With timeout: %s", c.timeout), func(t *testing.T) { | ||||
| 			config := protecodeExecuteScanOptions{TimeoutMinutes: c.timeout} | ||||
|  | ||||
| 		client := createClient(&config) | ||||
| 		assert.NotNil(t, client, "client should not be empty") | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestCreateDockerClient(t *testing.T) { | ||||
| 	cases := []struct { | ||||
| 		scanImage         string | ||||
| 		dockerRegistryURL string | ||||
| 		filePath          string | ||||
| 		includeLayers     bool | ||||
| 	}{ | ||||
| 		{"test", "url", "path", false}, | ||||
| 		{"", "", "", true}, | ||||
| 	} | ||||
|  | ||||
| 	for _, c := range cases { | ||||
| 		config := protecodeExecuteScanOptions{ScanImage: c.scanImage, DockerRegistryURL: c.dockerRegistryURL, FilePath: c.filePath, IncludeLayers: c.includeLayers} | ||||
| 		client := createDockerClient(&config) | ||||
| 		assert.NotNil(t, client, "client should not be empty") | ||||
| 			client := createProtecodeClient(&config) | ||||
| 			assert.NotNil(t, client, "client should not be empty") | ||||
| 		}) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| @@ -270,18 +250,21 @@ func TestUploadScanOrDeclareFetch(t *testing.T) { | ||||
| 		{false, "test", "group1", "", path, "", 0, 4711}, | ||||
| 	} | ||||
|  | ||||
| 	for _, c := range cases { | ||||
| 		// test | ||||
| 		config := protecodeExecuteScanOptions{VerifyOnly: c.reuse, CleanupMode: c.clean, Group: c.group, FetchURL: c.fetchURL, FilePath: c.filePath} | ||||
| 		// got := uploadScanOrDeclareFetch(config, 0, pc, fileName) | ||||
| 		got := uploadScanOrDeclareFetch(config, c.prID, pc, fileName) | ||||
| 		// assert | ||||
| 		assert.Equal(t, c.want, got) | ||||
| 	} | ||||
| } | ||||
| 	for i, c := range cases { | ||||
| 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { | ||||
| 			utils := protecodeTestUtilsBundle{ | ||||
| 				FilesMock:    &mock.FilesMock{}, | ||||
| 				DownloadMock: &mock.DownloadMock{}, | ||||
| 			} | ||||
|  | ||||
| func writeReportToFileMock(resp io.ReadCloser, reportFileName string) error { | ||||
| 	return nil | ||||
| 			// test | ||||
| 			config := protecodeExecuteScanOptions{VerifyOnly: c.reuse, CleanupMode: c.clean, Group: c.group, FetchURL: c.fetchURL, FilePath: c.filePath} | ||||
| 			// got := uploadScanOrDeclareFetch(config, 0, pc, fileName) | ||||
| 			got := uploadScanOrDeclareFetch(utils, config, c.prID, pc, fileName) | ||||
| 			// assert | ||||
| 			assert.Equal(t, c.want, got) | ||||
| 		}) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestExecuteProtecodeScan(t *testing.T) { | ||||
| @@ -333,26 +316,23 @@ func TestExecuteProtecodeScan(t *testing.T) { | ||||
| 	require.NoErrorf(t, err, "Failed to get current directory: %v", err) | ||||
| 	defer func() { _ = os.Chdir(resetDir) }() | ||||
|  | ||||
| 	for _, c := range cases { | ||||
| 		// init | ||||
| 		dir, err := ioutil.TempDir("", "t") | ||||
| 		require.NoErrorf(t, err, "Failed to create temporary directory: %v", err) | ||||
| 		// clean up tmp dir | ||||
| 		defer func() { _ = os.RemoveAll(dir) }() | ||||
| 		// change into tmp dir and write test data | ||||
| 		err = os.Chdir(dir) | ||||
| 		require.NoErrorf(t, err, "Failed to change into temporary directory: %v", err) | ||||
| 		reportPath = dir | ||||
| 		config := protecodeExecuteScanOptions{VerifyOnly: c.reuse, CleanupMode: c.clean, Group: c.group, FetchURL: c.fetchURL, TimeoutMinutes: "3", ExcludeCVEs: "CVE-2018-1, CVE-2017-1000382", ReportFileName: "./cache/report-file.txt"} | ||||
| 		influxData := &protecodeExecuteScanInflux{} | ||||
| 		// test | ||||
| 		executeProtecodeScan(influxData, pc, &config, "dummy", writeReportToFileMock) | ||||
| 		// assert | ||||
| 		assert.Equal(t, 1125, influxData.protecode_data.fields.historical_vulnerabilities) | ||||
| 		assert.Equal(t, 0, influxData.protecode_data.fields.triaged_vulnerabilities) | ||||
| 		assert.Equal(t, 1, influxData.protecode_data.fields.excluded_vulnerabilities) | ||||
| 		assert.Equal(t, 142, influxData.protecode_data.fields.major_vulnerabilities) | ||||
| 		assert.Equal(t, 226, influxData.protecode_data.fields.vulnerabilities) | ||||
| 	for i, c := range cases { | ||||
| 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { | ||||
| 			utils := protecodeTestUtilsBundle{ | ||||
| 				FilesMock:    &mock.FilesMock{}, | ||||
| 				DownloadMock: &mock.DownloadMock{}, | ||||
| 			} | ||||
| 			config := protecodeExecuteScanOptions{VerifyOnly: c.reuse, CleanupMode: c.clean, Group: c.group, FetchURL: c.fetchURL, TimeoutMinutes: "3", ExcludeCVEs: "CVE-2018-1, CVE-2017-1000382", ReportFileName: "./cache/report-file.txt"} | ||||
| 			influxData := &protecodeExecuteScanInflux{} | ||||
| 			// test | ||||
| 			executeProtecodeScan(influxData, pc, &config, "dummy", utils) | ||||
| 			// assert | ||||
| 			assert.Equal(t, 1125, influxData.protecode_data.fields.historical_vulnerabilities) | ||||
| 			assert.Equal(t, 0, influxData.protecode_data.fields.triaged_vulnerabilities) | ||||
| 			assert.Equal(t, 1, influxData.protecode_data.fields.excluded_vulnerabilities) | ||||
| 			assert.Equal(t, 142, influxData.protecode_data.fields.major_vulnerabilities) | ||||
| 			assert.Equal(t, 226, influxData.protecode_data.fields.vulnerabilities) | ||||
| 		}) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| @@ -390,42 +370,3 @@ func TestCorrectDockerConfigEnvVar(t *testing.T) { | ||||
| 		assert.Equal(t, resetValue, os.Getenv("DOCKER_CONFIG")) | ||||
| 	}) | ||||
| } | ||||
|  | ||||
| func TestGetTarName(t *testing.T) { | ||||
| 	cases := map[string]struct { | ||||
| 		image   string | ||||
| 		version string | ||||
| 		expect  string | ||||
| 	}{ | ||||
| 		"with version suffix": { | ||||
| 			"com.sap.piper/sample-k8s-app-multistage:1.11-20200902040158_97a5cc34f1796ad735159f020dd55c0f3670a4cb", | ||||
| 			"1.11-20200902040158_97a5cc34f1796ad735159f020dd55c0f3670a4cb", | ||||
| 			"com.sap.piper_sample-k8s-app-multistage_1.tar", | ||||
| 		}, | ||||
| 		"without version suffix": { | ||||
| 			"abc", | ||||
| 			"3.20.20-20200131085038+eeb7c1033339bfd404d21ec5e7dc05c80e9e985e", | ||||
| 			"abc_3.tar", | ||||
| 		}, | ||||
| 		"without version": { | ||||
| 			"abc", | ||||
| 			"", | ||||
| 			"abc.tar", | ||||
| 		}, | ||||
| 		"ScanImage without sha as artifactVersion": { | ||||
| 			"abc@sha256:12345", | ||||
| 			"", | ||||
| 			"abc.tar", | ||||
| 		}, | ||||
| 		"ScanImage with sha as artifactVersion": { | ||||
| 			"ppiper/cf-cli@sha256:c25dbacb9ab6e912afe0fe926d8f9d949c60adfe55d16778bde5941e6c37be11", | ||||
| 			"c25dbacb9ab6e912afe0fe926d8f9d949c60adfe55d16778bde5941e6c37be11", | ||||
| 			"ppiper_cf-cli_c25dbacb9ab6e912afe0fe926d8f9d949c60adfe55d16778bde5941e6c37be11.tar", | ||||
| 		}, | ||||
| 	} | ||||
| 	for name, c := range cases { | ||||
| 		t.Run(name, func(t *testing.T) { | ||||
| 			assert.Equal(t, c.expect, getTarName(&protecodeExecuteScanOptions{ScanImage: c.image, Version: c.version})) | ||||
| 		}) | ||||
| 	} | ||||
| } | ||||
|   | ||||
							
								
								
									
										14
									
								
								cmd/utils.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										14
									
								
								cmd/utils.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,14 @@ | ||||
| package cmd | ||||
|  | ||||
| import ( | ||||
| 	"os" | ||||
| ) | ||||
|  | ||||
| // Deprecated: Please use piperutils.Files{} instead | ||||
| func fileExists(filename string) bool { | ||||
| 	info, err := os.Stat(filename) | ||||
| 	if os.IsNotExist(err) { | ||||
| 		return false | ||||
| 	} | ||||
| 	return !info.IsDir() | ||||
| } | ||||
| @@ -177,10 +177,9 @@ func runWhitesourceScan(config *ScanOptions, scan *ws.Scan, utils whitesourceUti | ||||
| 			ContainerRegistryUser:     config.ContainerRegistryUser, | ||||
| 			ContainerRegistryPassword: config.ContainerRegistryPassword, | ||||
| 			DockerConfigJSON:          config.DockerConfigJSON, | ||||
| 			IncludeLayers:             config.ScanImageIncludeLayers, | ||||
| 			FilePath:                  config.ProjectName, | ||||
| 		} | ||||
| 		dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", IncludeLayers: saveImageOptions.IncludeLayers} | ||||
| 		dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: ""} | ||||
| 		dClient := &piperDocker.Client{} | ||||
| 		dClient.SetOptions(dClientOptions) | ||||
| 		if _, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient, utils); err != nil { | ||||
|   | ||||
| @@ -52,7 +52,6 @@ type whitesourceExecuteScanOptions struct { | ||||
| 	ProjectToken                         string   `json:"projectToken,omitempty"` | ||||
| 	Reporting                            bool     `json:"reporting,omitempty"` | ||||
| 	ScanImage                            string   `json:"scanImage,omitempty"` | ||||
| 	ScanImageIncludeLayers               bool     `json:"scanImageIncludeLayers,omitempty"` | ||||
| 	ScanImageRegistryURL                 string   `json:"scanImageRegistryUrl,omitempty"` | ||||
| 	SecurityVulnerabilities              bool     `json:"securityVulnerabilities,omitempty"` | ||||
| 	ServiceURL                           string   `json:"serviceUrl,omitempty"` | ||||
| @@ -331,7 +330,6 @@ func addWhitesourceExecuteScanFlags(cmd *cobra.Command, stepConfig *whitesourceE | ||||
| 	cmd.Flags().StringVar(&stepConfig.ProjectToken, "projectToken", os.Getenv("PIPER_projectToken"), "Project token to execute scan on. Ignored for scan types `maven`, `mta` and `npm`. Used for project aggregation when scanning with the Unified Agent and can be provided as an alternative to `projectName`.") | ||||
| 	cmd.Flags().BoolVar(&stepConfig.Reporting, "reporting", true, "Whether assessment is being done at all, defaults to `true`") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ScanImage, "scanImage", os.Getenv("PIPER_scanImage"), "For `buildTool: docker`: Defines the docker image which should be scanned.") | ||||
| 	cmd.Flags().BoolVar(&stepConfig.ScanImageIncludeLayers, "scanImageIncludeLayers", true, "For `buildTool: docker`: Defines if layers should be included.") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ScanImageRegistryURL, "scanImageRegistryUrl", os.Getenv("PIPER_scanImageRegistryUrl"), "For `buildTool: docker`: Defines the registry where the scanImage is located.") | ||||
| 	cmd.Flags().BoolVar(&stepConfig.SecurityVulnerabilities, "securityVulnerabilities", true, "Whether security compliance is considered and reported as part of the assessment.") | ||||
| 	cmd.Flags().StringVar(&stepConfig.ServiceURL, "serviceUrl", `https://saas.whitesourcesoftware.com/api`, "URL to the WhiteSource API endpoint.") | ||||
| @@ -712,15 +710,6 @@ func whitesourceExecuteScanMetadata() config.StepData { | ||||
| 						Aliases:   []config.Alias{}, | ||||
| 						Default:   os.Getenv("PIPER_scanImage"), | ||||
| 					}, | ||||
| 					{ | ||||
| 						Name:        "scanImageIncludeLayers", | ||||
| 						ResourceRef: []config.ResourceReference{}, | ||||
| 						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"}, | ||||
| 						Type:        "bool", | ||||
| 						Mandatory:   false, | ||||
| 						Aliases:     []config.Alias{}, | ||||
| 						Default:     true, | ||||
| 					}, | ||||
| 					{ | ||||
| 						Name: "scanImageRegistryUrl", | ||||
| 						ResourceRef: []config.ResourceReference{ | ||||
|   | ||||
							
								
								
									
										1
									
								
								go.mod
									
									
									
									
									
								
							
							
						
						
									
										1
									
								
								go.mod
									
									
									
									
									
								
							| @@ -4,7 +4,6 @@ go 1.17 | ||||
|  | ||||
| require ( | ||||
| 	cloud.google.com/go/storage v1.10.0 | ||||
| 	github.com/GoogleContainerTools/container-diff v0.17.0 | ||||
| 	github.com/Jeffail/gabs/v2 v2.6.1 | ||||
| 	github.com/Masterminds/sprig v2.22.0+incompatible | ||||
| 	github.com/antchfx/htmlquery v1.2.4 | ||||
|   | ||||
							
								
								
									
										17
									
								
								go.sum
									
									
									
									
									
								
							
							
						
						
									
										17
									
								
								go.sum
									
									
									
									
									
								
							| @@ -63,7 +63,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl | ||||
| cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= | ||||
| cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= | ||||
| cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= | ||||
| code.cloudfoundry.org/bytefmt v0.0.0-20180906201452-2aa6f33b730c/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= | ||||
| code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f h1:UrKzEwTgeiff9vxdrfdqxibzpWjxLnuXDI5m6z3GJAk= | ||||
| code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f/go.mod h1:sk5LnIjB/nIEU7yP5sDQExVm62wu0pBh3yrElngUisI= | ||||
| dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= | ||||
| @@ -151,8 +150,6 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q | ||||
| github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= | ||||
| github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dXCilEuNEeAn20fdD4= | ||||
| github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= | ||||
| github.com/GoogleContainerTools/container-diff v0.17.0 h1:P9va/peQsiBcFuL+hcZWHp3Ry7bCLLoNhUrjj7DBNFQ= | ||||
| github.com/GoogleContainerTools/container-diff v0.17.0/go.mod h1:pddOdZb5Wghtb2fmqbKy7Kj/sgoafW9AvUp4KS9svmg= | ||||
| github.com/Jeffail/gabs v1.1.1 h1:V0uzR08Hj22EX8+8QMhyI9sX2hwRu+/RJhJUmnwda/E= | ||||
| github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= | ||||
| github.com/Jeffail/gabs/v2 v2.6.1 h1:wwbE6nTQTwIMsMxzi6XFQQYRZ6wDc1mSdxoAN+9U4Gk= | ||||
| @@ -448,7 +445,6 @@ github.com/containerd/containerd v1.5.2/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTV | ||||
| github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= | ||||
| github.com/containerd/containerd v1.5.9 h1:rs6Xg1gtIxaeyG+Smsb/0xaSDu1VgFhOCKBXxMxbsF4= | ||||
| github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= | ||||
| github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= | ||||
| github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= | ||||
| github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= | ||||
| github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= | ||||
| @@ -583,12 +579,9 @@ github.com/docker/cli v20.10.9+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHv | ||||
| github.com/docker/cli v20.10.11+incompatible h1:tXU1ezXcruZQRrMP8RN2z9N91h+6egZTS1gsPsKantc= | ||||
| github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= | ||||
| github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= | ||||
| github.com/docker/distribution v0.0.0-20200319173657-742aab907b54/go.mod h1:Oqz4IonmMNc2N7GqfTL2xkhCQx0yS6nR+HrOZJnmKIk= | ||||
| github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= | ||||
| github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= | ||||
| github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= | ||||
| github.com/docker/docker v0.7.3-0.20190212235812-0111ee70874a/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= | ||||
| github.com/docker/docker v1.4.2-0.20190219180918-740349757396/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= | ||||
| github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= | ||||
| github.com/docker/docker v1.4.2-0.20200319182547-c7ad2b866182/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= | ||||
| github.com/docker/docker v20.10.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= | ||||
| @@ -683,7 +676,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo | ||||
| github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= | ||||
| github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= | ||||
| github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= | ||||
| github.com/fsouza/go-dockerclient v1.3.6/go.mod h1:ptN6nXBwrXuiHAz2TYGOFCBB1aKGr371sGjMFdJEr1A= | ||||
| github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= | ||||
| github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= | ||||
| github.com/gabriel-vasile/mimetype v1.3.1 h1:qevA6c2MtE1RorlScnixeG0VA1H4xrXyhyX3oWBynNQ= | ||||
| @@ -983,7 +975,6 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ | ||||
| github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= | ||||
| github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= | ||||
| github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= | ||||
| github.com/google/go-containerregistry v0.0.0-20190214194807-bada66e31e55/go.mod h1:yZAFP63pRshzrEYLXLGPmUt0Ay+2zdjmMN1loCnRLUk= | ||||
| github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= | ||||
| github.com/google/go-containerregistry v0.5.2-0.20210604130445-3bfab55f3bd9/go.mod h1:R5WRYyTdQqTchlBhX4q+WICGh8HQIL5wDFoFZv7Jq6Q= | ||||
| github.com/google/go-containerregistry v0.6.0 h1:niQ+8XD//kKgArIFwDVBXsWVWbde16LPdHMyNwSC8h4= | ||||
| @@ -1051,7 +1042,6 @@ github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu | ||||
| github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= | ||||
| github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= | ||||
| github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= | ||||
| github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= | ||||
| github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= | ||||
| github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= | ||||
| github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= | ||||
| @@ -1312,7 +1302,6 @@ github.com/huaweicloud/golangsdk v0.0.0-20200304081349-45ec0797f2a4/go.mod h1:WQ | ||||
| github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= | ||||
| github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= | ||||
| github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= | ||||
| github.com/ijc/Gotty v0.0.0-20170406111628-a8b993ba6abd/go.mod h1:3LVOLeyx9XVvwPgrt2be44XgSqndprz1G18rSk8KD84= | ||||
| github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= | ||||
| github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= | ||||
| github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= | ||||
| @@ -1639,7 +1628,6 @@ github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ | ||||
| github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2 h1:BQ1HW7hr4IVovMwWg0E0PYcyW8CzqDcVmaew9cujU4s= | ||||
| github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h1:TLb2Sg7HQcgGdloNxkrmtgDNR9uVYF3lfdFIN4Ro6Sk= | ||||
| github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= | ||||
| github.com/nightlyone/lockfile v0.0.0-20180618180623-0ad87eef1443/go.mod h1:JbxfV1Iifij2yhRjXai0oFrbpxszXHRx1E5RuM26o4Y= | ||||
| github.com/nwaples/rardecode v1.1.2/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= | ||||
| github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= | ||||
| github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= | ||||
| @@ -1765,7 +1753,6 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV | ||||
| github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= | ||||
| github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= | ||||
| github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= | ||||
| github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= | ||||
| github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= | ||||
| github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= | ||||
| github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | ||||
| @@ -1882,7 +1869,6 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV | ||||
| github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= | ||||
| github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= | ||||
| github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= | ||||
| github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= | ||||
| github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= | ||||
| github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= | ||||
| github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= | ||||
| @@ -2128,7 +2114,6 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf | ||||
| golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= | ||||
| golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= | ||||
| golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= | ||||
| golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= | ||||
| golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= | ||||
| golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= | ||||
| golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= | ||||
| @@ -2150,7 +2135,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U | ||||
| golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | ||||
| golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | ||||
| golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | ||||
| golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | ||||
| golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | ||||
| golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | ||||
| golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | ||||
| @@ -2343,7 +2327,6 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h | ||||
| golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||||
| golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||||
| golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||||
| golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||||
| golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||||
| golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||||
| golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||||
|   | ||||
| @@ -46,12 +46,12 @@ func DownloadBuildpacks(path string, bpacks []string, dockerCreds string, utils | ||||
| 		defer utils.RemoveAll(tempDir) | ||||
|  | ||||
| 		log.Entry().Infof("Downloading buildpack '%s' to %s", bpack, tempDir) | ||||
| 		img, err := utils.DownloadImageToPath(bpack, tempDir) | ||||
| 		img, err := utils.DownloadImageContent(bpack, tempDir) | ||||
| 		if err != nil { | ||||
| 			return Order{}, fmt.Errorf("failed download buildpack image '%s', error: %s", bpack, err.Error()) | ||||
| 		} | ||||
|  | ||||
| 		imgConf, err := img.Image.ConfigFile() | ||||
| 		imgConf, err := img.ConfigFile() | ||||
| 		if err != nil { | ||||
| 			return Order{}, fmt.Errorf("failed to read '%s' image config, error: %s", bpack, err.Error()) | ||||
| 		} | ||||
|   | ||||
| @@ -4,10 +4,9 @@ | ||||
| package cnbutils | ||||
|  | ||||
| import ( | ||||
| 	"io" | ||||
| 	"fmt" | ||||
| 	"path/filepath" | ||||
|  | ||||
| 	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util" | ||||
| 	"github.com/SAP/jenkins-library/pkg/mock" | ||||
| 	"github.com/SAP/jenkins-library/pkg/piperutils" | ||||
| 	v1 "github.com/google/go-containerregistry/pkg/v1" | ||||
| @@ -23,7 +22,7 @@ func (c *MockUtils) GetFileUtils() piperutils.FileUtils { | ||||
| 	return c.FilesMock | ||||
| } | ||||
|  | ||||
| func (c *MockUtils) DownloadImageToPath(bpack, filePath string) (pkgutil.Image, error) { | ||||
| func (c *MockUtils) DownloadImageContent(bpack, targetDir string) (v1.Image, error) { | ||||
| 	fakeImage := fakeImage.FakeImage{} | ||||
| 	fakeImage.ConfigFileReturns(&v1.ConfigFile{ | ||||
| 		Config: v1.Config{ | ||||
| @@ -33,18 +32,15 @@ func (c *MockUtils) DownloadImageToPath(bpack, filePath string) (pkgutil.Image, | ||||
| 		}, | ||||
| 	}, nil) | ||||
|  | ||||
| 	c.AddDir(filepath.Join(filePath, "cnb/buildpacks", bpack)) | ||||
| 	c.AddDir(filepath.Join(filePath, "cnb/buildpacks", bpack, "0.0.1")) | ||||
| 	img := pkgutil.Image{ | ||||
| 		Image: &fakeImage, | ||||
| 	} | ||||
| 	return img, nil | ||||
| 	c.AddDir(filepath.Join(targetDir, "cnb/buildpacks", bpack)) | ||||
| 	c.AddDir(filepath.Join(targetDir, "cnb/buildpacks", bpack, "0.0.1")) | ||||
| 	return &fakeImage, nil | ||||
| } | ||||
|  | ||||
| func (c *MockUtils) DownloadImage(src, dst string) (v1.Image, error) { | ||||
| 	return nil, fmt.Errorf("not implemented") | ||||
| } | ||||
|  | ||||
| func (c *MockUtils) GetImageSource() (string, error) { | ||||
| 	return "imageSource", nil | ||||
| } | ||||
|  | ||||
| func (c *MockUtils) TarImage(writer io.Writer, image pkgutil.Image) error { | ||||
| 	return nil | ||||
| } | ||||
|   | ||||
| @@ -4,17 +4,18 @@ import ( | ||||
| 	"encoding/base64" | ||||
| 	"encoding/json" | ||||
| 	"fmt" | ||||
| 	"io" | ||||
| 	"net/url" | ||||
| 	"os" | ||||
| 	"path/filepath" | ||||
| 	"regexp" | ||||
| 	"strings" | ||||
|  | ||||
| 	"github.com/SAP/jenkins-library/pkg/log" | ||||
| 	"github.com/SAP/jenkins-library/pkg/piperutils" | ||||
|  | ||||
| 	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util" | ||||
| 	"github.com/google/go-containerregistry/pkg/legacy/tarball" | ||||
| 	cranecmd "github.com/google/go-containerregistry/cmd/crane/cmd" | ||||
| 	"github.com/google/go-containerregistry/pkg/crane" | ||||
| 	"github.com/google/go-containerregistry/pkg/name" | ||||
| 	"github.com/google/go-containerregistry/pkg/v1" | ||||
| ) | ||||
|  | ||||
| // AuthEntry defines base64 encoded username:password required inside a Docker config.json | ||||
| @@ -84,87 +85,109 @@ type Client struct { | ||||
|  | ||||
| // ClientOptions defines the options to be set on the client | ||||
| type ClientOptions struct { | ||||
| 	ImageName     string | ||||
| 	RegistryURL   string | ||||
| 	LocalPath     string | ||||
| 	IncludeLayers bool | ||||
| 	ImageName   string | ||||
| 	RegistryURL string | ||||
| 	LocalPath   string | ||||
| } | ||||
|  | ||||
| //Download interface for download an image to a local path | ||||
| type Download interface { | ||||
| 	GetImageSource() (string, error) | ||||
| 	DownloadImageToPath(imageSource, filePath string) (pkgutil.Image, error) | ||||
| 	TarImage(writer io.Writer, image pkgutil.Image) error | ||||
| 	DownloadImage(imageSource, targetFile string) (v1.Image, error) | ||||
| 	DownloadImageContent(imageSource, targetDir string) (v1.Image, error) | ||||
| } | ||||
|  | ||||
| // SetOptions sets options used for the docker client | ||||
| func (c *Client) SetOptions(options ClientOptions) { | ||||
| 	c.imageName = options.ImageName | ||||
| 	c.registryURL = options.RegistryURL | ||||
| 	c.includeLayers = options.IncludeLayers | ||||
| 	c.localPath = options.LocalPath | ||||
| } | ||||
|  | ||||
| const ( | ||||
| 	daemonPrefix = "daemon://" | ||||
| 	remotePrefix = "remote://" | ||||
| ) | ||||
|  | ||||
| //GetImageSource get the image source from client attributes (localPath, imageName, registryURL) | ||||
| func (c *Client) GetImageSource() (string, error) { | ||||
|  | ||||
| 	imageSource := c.imageName | ||||
|  | ||||
| 	if len(c.registryURL) > 0 && len(c.localPath) <= 0 { | ||||
| 		registry := c.registryURL | ||||
|  | ||||
| 		url, err := url.Parse(c.registryURL) | ||||
| 		if err != nil { | ||||
| 			return "", fmt.Errorf("failed to parse registryURL %v: %w", c.registryURL, err) | ||||
| 		} | ||||
|  | ||||
| 		//remove protocol from registryURL to get registry | ||||
| 		if len(url.Scheme) > 0 { | ||||
| 			registry = strings.Replace(c.registryURL, fmt.Sprintf("%v://", url.Scheme), "", 1) | ||||
| 		} | ||||
|  | ||||
| 		if strings.HasSuffix(registry, "/") { | ||||
| 			imageSource = fmt.Sprintf("%v%v%v", remotePrefix, registry, c.imageName) | ||||
| 		} else { | ||||
| 			imageSource = fmt.Sprintf("%v%v/%v", remotePrefix, registry, c.imageName) | ||||
| 		} | ||||
| 	} else if len(c.localPath) > 0 { | ||||
| 		imageSource = c.localPath | ||||
| 		if !pkgutil.IsTar(c.localPath) { | ||||
| 			imageSource = fmt.Sprintf("%v%v", daemonPrefix, c.localPath) | ||||
| 		} | ||||
| //DownloadImageToPath downloads the image content into the given targetDir. Returns with an error if the targetDir doesnt exist | ||||
| func (c *Client) DownloadImageContent(imageSource, targetDir string) (v1.Image, error) { | ||||
| 	if fileInfo, err := os.Stat(targetDir); err != nil { | ||||
| 		return nil, err | ||||
| 	} else if !fileInfo.IsDir() { | ||||
| 		return nil, fmt.Errorf("specified target is not a directory: %s", targetDir) | ||||
| 	} | ||||
|  | ||||
| 	if len(imageSource) <= 0 { | ||||
| 		return imageSource, fmt.Errorf("no image found for the parameters: (Name: %v, Registry: %v, local Path: %v)", c.imageName, c.registryURL, c.localPath) | ||||
| 	noOpts := []crane.Option{} | ||||
|  | ||||
| 	imageRef, err := c.getImageRef(imageSource) | ||||
| 	if err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	return imageSource, nil | ||||
| 	img, err := crane.Pull(imageRef.Name(), noOpts...) | ||||
| 	if err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	tmpFile, err := os.CreateTemp(".", ".piper-download-") | ||||
| 	if err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
| 	defer os.Remove(tmpFile.Name()) | ||||
|  | ||||
| 	args := []string{imageRef.Name(), tmpFile.Name()} | ||||
|  | ||||
| 	exportCmd := cranecmd.NewCmdExport(&noOpts) | ||||
| 	exportCmd.SetArgs(args) | ||||
|  | ||||
| 	if err := exportCmd.Execute(); err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	return img, piperutils.Untar(tmpFile.Name(), targetDir, 0) | ||||
| } | ||||
|  | ||||
| //DownloadImageToPath download the image to the specified path | ||||
| func (c *Client) DownloadImageToPath(imageSource, filePath string) (pkgutil.Image, error) { | ||||
| // DownloadImage downloads the image and saves it as tar at the given path | ||||
| func (c *Client) DownloadImage(imageSource, targetFile string) (v1.Image, error) { | ||||
| 	noOpts := []crane.Option{} | ||||
|  | ||||
| 	return pkgutil.GetImage(imageSource, c.includeLayers, filePath) | ||||
| 	imageRef, err := c.getImageRef(imageSource) | ||||
| 	if err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	img, err := crane.Pull(imageRef.Name(), noOpts...) | ||||
| 	if err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	tmpFile, err := os.CreateTemp(".", ".piper-download-") | ||||
| 	if err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	craneCmd := cranecmd.NewCmdPull(&noOpts) | ||||
| 	craneCmd.SetOut(log.Writer()) | ||||
| 	craneCmd.SetErr(log.Writer()) | ||||
| 	craneCmd.SetArgs([]string{imageRef.Name(), tmpFile.Name(), "--format=tarball"}) | ||||
|  | ||||
| 	if err := craneCmd.Execute(); err != nil { | ||||
| 		defer os.Remove(tmpFile.Name()) | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	if err := os.Rename(tmpFile.Name(), targetFile); err != nil { | ||||
| 		defer os.Remove(tmpFile.Name()) | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	return img, nil | ||||
| } | ||||
|  | ||||
| //TarImage write a tar from the given image | ||||
| func (c *Client) TarImage(writer io.Writer, image pkgutil.Image) error { | ||||
| func (c *Client) getImageRef(image string) (name.Reference, error) { | ||||
| 	opts := []name.Option{} | ||||
|  | ||||
| 	reference, err := name.ParseReference(image.Digest.String(), name.WeakValidation) | ||||
| 	if err != nil { | ||||
| 		return err | ||||
| 	if len(c.registryURL) > 0 { | ||||
| 		re := regexp.MustCompile(`(?i)^https?://`) | ||||
| 		registry := re.ReplaceAllString(c.registryURL, "") | ||||
| 		opts = append(opts, name.WithDefaultRegistry(registry)) | ||||
| 	} | ||||
| 	err = tarball.Write(reference, image.Image, writer) | ||||
| 	if err != nil { | ||||
| 		return err | ||||
| 	} | ||||
| 	return nil | ||||
|  | ||||
| 	return name.ParseReference(image, opts...) | ||||
| } | ||||
|  | ||||
| // ImageListWithFilePath compiles container image names based on all Dockerfiles found, considering excludes | ||||
|   | ||||
| @@ -103,35 +103,6 @@ func TestCreateDockerConfigJSON(t *testing.T) { | ||||
| 	}) | ||||
| } | ||||
|  | ||||
| func TestGetImageSource(t *testing.T) { | ||||
|  | ||||
| 	cases := []struct { | ||||
| 		imageName   string | ||||
| 		registryURL string | ||||
| 		localPath   string | ||||
| 		want        string | ||||
| 	}{ | ||||
| 		{"imageName", "", "", "imageName"}, | ||||
| 		{"imageName", "", "localPath", "daemon://localPath"}, | ||||
| 		{"imageName", "http://registryURL", "", "remote://registryURL/imageName"}, | ||||
| 		{"imageName", "https://containerRegistryUrl", "", "remote://containerRegistryUrl/imageName"}, | ||||
| 		{"imageName", "registryURL", "", "remote://registryURL/imageName"}, | ||||
| 	} | ||||
|  | ||||
| 	client := Client{} | ||||
|  | ||||
| 	for _, c := range cases { | ||||
|  | ||||
| 		options := ClientOptions{ImageName: c.imageName, RegistryURL: c.registryURL, LocalPath: c.localPath} | ||||
| 		client.SetOptions(options) | ||||
|  | ||||
| 		got, err := client.GetImageSource() | ||||
|  | ||||
| 		assert.Nil(t, err) | ||||
| 		assert.Equal(t, c.want, got) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestImageListWithFilePath(t *testing.T) { | ||||
| 	t.Parallel() | ||||
|  | ||||
|   | ||||
| @@ -13,7 +13,6 @@ import ( | ||||
| 	"mime/multipart" | ||||
| 	"net" | ||||
| 	"net/http" | ||||
| 	"os" | ||||
| 	"path" | ||||
| 	"path/filepath" | ||||
| 	"strings" | ||||
| @@ -42,6 +41,7 @@ type Client struct { | ||||
| 	doLogResponseBodyOnDebug  bool | ||||
| 	useDefaultTransport       bool | ||||
| 	trustedCerts              []string | ||||
| 	fileUtils                 piperutils.FileUtils | ||||
| } | ||||
|  | ||||
| // ClientOptions defines the options to be set on the client | ||||
| @@ -110,6 +110,15 @@ type Uploader interface { | ||||
| 	Upload(data UploadRequestData) (*http.Response, error) | ||||
| } | ||||
|  | ||||
| // fileUtils lazy initializes the utils | ||||
| func (c *Client) getFileUtils() piperutils.FileUtils { | ||||
| 	if c.fileUtils == nil { | ||||
| 		c.fileUtils = &piperutils.Files{} | ||||
| 	} | ||||
|  | ||||
| 	return c.fileUtils | ||||
| } | ||||
|  | ||||
| // UploadFile uploads a file's content as multipart-form POST request to the specified URL | ||||
| func (c *Client) UploadFile(url, file, fileFieldName string, header http.Header, cookies []*http.Cookie, uploadType string) (*http.Response, error) { | ||||
| 	return c.UploadRequest(http.MethodPost, url, file, fileFieldName, header, cookies, uploadType) | ||||
| @@ -117,7 +126,8 @@ func (c *Client) UploadFile(url, file, fileFieldName string, header http.Header, | ||||
|  | ||||
| // UploadRequest uploads a file's content as multipart-form with given http method request to the specified URL | ||||
| func (c *Client) UploadRequest(method, url, file, fileFieldName string, header http.Header, cookies []*http.Cookie, uploadType string) (*http.Response, error) { | ||||
| 	fileHandle, err := os.Open(file) | ||||
| 	fileHandle, err := c.getFileUtils().Open(file) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		return &http.Response{}, errors.Wrapf(err, "unable to locate file %v", file) | ||||
| 	} | ||||
| @@ -245,6 +255,12 @@ func (c *Client) SetOptions(options ClientOptions) { | ||||
| 	} | ||||
| 	c.cookieJar = options.CookieJar | ||||
| 	c.trustedCerts = options.TrustedCerts | ||||
| 	c.fileUtils = &piperutils.Files{} | ||||
| } | ||||
|  | ||||
| // SetFileUtils can be used to overwrite the default file utils | ||||
| func (c *Client) SetFileUtils(fileUtils piperutils.FileUtils) { | ||||
| 	c.fileUtils = fileUtils | ||||
| } | ||||
|  | ||||
| // StandardClient returns a stdlib *http.Client which respects the custom settings. | ||||
| @@ -500,7 +516,6 @@ func (c *Client) applyDefaults() { | ||||
| func (c *Client) configureTLSToTrustCertificates(transport *TransportWrapper) error { | ||||
|  | ||||
| 	trustStoreDir, err := getWorkingDirForTrustStore() | ||||
| 	fileUtils := &piperutils.Files{} | ||||
| 	if err != nil { | ||||
| 		return errors.Wrap(err, "failed to create trust store directory") | ||||
| 	} | ||||
| @@ -539,7 +554,7 @@ func (c *Client) configureTLSToTrustCertificates(transport *TransportWrapper) er | ||||
| 		filename := path.Base(certificate) | ||||
| 		filename = strings.ReplaceAll(filename, " ", "") | ||||
| 		target := filepath.Join(trustStoreDir, filename) | ||||
| 		if exists, _ := fileUtils.FileExists(target); !exists { | ||||
| 		if exists, _ := c.getFileUtils().FileExists(target); !exists { | ||||
| 			log.Entry().WithField("source", certificate).WithField("target", target).Info("Downloading TLS certificate") | ||||
| 			request, err := http.NewRequest("GET", certificate, nil) | ||||
| 			if err != nil { | ||||
| @@ -561,11 +576,11 @@ func (c *Client) configureTLSToTrustCertificates(transport *TransportWrapper) er | ||||
| 				defer response.Body.Close() | ||||
| 				parent := filepath.Dir(target) | ||||
| 				if len(parent) > 0 { | ||||
| 					if err = os.MkdirAll(parent, 0777); err != nil { | ||||
| 					if err = c.getFileUtils().MkdirAll(parent, 0777); err != nil { | ||||
| 						return err | ||||
| 					} | ||||
| 				} | ||||
| 				fileHandler, err := os.Create(target) | ||||
| 				fileHandler, err := c.getFileUtils().Create(target) | ||||
| 				if err != nil { | ||||
| 					return errors.Wrapf(err, "unable to create file %v", filename) | ||||
| 				} | ||||
|   | ||||
							
								
								
									
										44
									
								
								pkg/mock/dockerClient.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										44
									
								
								pkg/mock/dockerClient.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,44 @@ | ||||
| package mock | ||||
|  | ||||
| import ( | ||||
| 	"fmt" | ||||
| 	"github.com/google/go-containerregistry/pkg/v1" | ||||
| ) | ||||
|  | ||||
| // DownloadMock . | ||||
| type DownloadMock struct { | ||||
| 	FilePath    string | ||||
| 	ImageRef    string | ||||
| 	RegistryURL string | ||||
|  | ||||
| 	ReturnImage v1.Image | ||||
| 	ReturnError string | ||||
|  | ||||
| 	Stub func(imageRef, targetDir string) (v1.Image, error) | ||||
| } | ||||
|  | ||||
| // DownloadImage . | ||||
| func (c *DownloadMock) DownloadImage(imageRef, targetDir string) (v1.Image, error) { | ||||
| 	c.ImageRef = imageRef | ||||
| 	c.FilePath = targetDir | ||||
|  | ||||
| 	if c.Stub != nil { | ||||
| 		return c.Stub(imageRef, targetDir) | ||||
| 	} | ||||
|  | ||||
| 	if len(c.ReturnError) > 0 { | ||||
| 		return nil, fmt.Errorf(c.ReturnError) | ||||
| 	} | ||||
| 	return c.ReturnImage, nil | ||||
| } | ||||
|  | ||||
| // DownloadImageContent . | ||||
| func (c *DownloadMock) DownloadImageContent(imageRef, targetFile string) (v1.Image, error) { | ||||
| 	c.ImageRef = imageRef | ||||
| 	c.FilePath = targetFile | ||||
|  | ||||
| 	if len(c.ReturnError) > 0 { | ||||
| 		return nil, fmt.Errorf(c.ReturnError) | ||||
| 	} | ||||
| 	return c.ReturnImage, nil | ||||
| } | ||||
| @@ -6,6 +6,7 @@ package mock | ||||
| import ( | ||||
| 	"crypto/sha256" | ||||
| 	"fmt" | ||||
| 	"io" | ||||
| 	"os" | ||||
| 	"path/filepath" | ||||
| 	"sort" | ||||
| @@ -538,11 +539,15 @@ type FileMock struct { | ||||
|  | ||||
| // Reads the content of the mock | ||||
| func (f *FileMock) Read(b []byte) (n int, err error) { | ||||
| 	if len(b) == 0 { | ||||
| 		return 0, nil | ||||
| 	} | ||||
|  | ||||
| 	for i, p := range f.content { | ||||
| 		b[i] = p | ||||
| 	} | ||||
|  | ||||
| 	return len(f.content), nil | ||||
| 	return len(f.content), io.EOF | ||||
| } | ||||
|  | ||||
| // Close mocks freeing the associated OS resources. | ||||
| @@ -610,3 +615,11 @@ func (f *FilesMock) OpenFile(path string, flag int, perm os.FileMode) (*FileMock | ||||
|  | ||||
| 	return &file, nil | ||||
| } | ||||
|  | ||||
| func (f *FilesMock) Open(name string) (io.ReadWriteCloser, error) { | ||||
| 	return f.OpenFile(name, os.O_RDONLY, 0) | ||||
| } | ||||
|  | ||||
| func (f *FilesMock) Create(name string) (io.ReadWriteCloser, error) { | ||||
| 	return f.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) | ||||
| } | ||||
|   | ||||
| @@ -8,6 +8,7 @@ import ( | ||||
| 	"errors" | ||||
| 	"fmt" | ||||
| 	"io" | ||||
| 	"io/fs" | ||||
| 	"io/ioutil" | ||||
| 	"os" | ||||
| 	"path/filepath" | ||||
| @@ -38,6 +39,8 @@ type FileUtils interface { | ||||
| 	Symlink(oldname string, newname string) error | ||||
| 	SHA256(path string) (string, error) | ||||
| 	CurrentTime(format string) string | ||||
| 	Open(name string) (io.ReadWriteCloser, error) | ||||
| 	Create(name string) (io.ReadWriteCloser, error) | ||||
| } | ||||
|  | ||||
| // Files ... | ||||
| @@ -230,20 +233,29 @@ func Unzip(src, dest string) ([]string, error) { | ||||
|  | ||||
| func Untar(src string, dest string, stripComponentLevel int) error { | ||||
| 	file, err := os.Open(src) | ||||
| 	defer file.Close() | ||||
|  | ||||
| 	if err != nil { | ||||
| 		fmt.Errorf("unable to open src: %v", err) | ||||
| 	} | ||||
|  | ||||
| 	if b, err := isFileGzipped(src); err != nil && b { | ||||
| 		zr, err := gzip.NewReader(file) | ||||
|  | ||||
| 		if err != nil { | ||||
| 			return fmt.Errorf("requires gzip-compressed body: %v", err) | ||||
| 		} | ||||
|  | ||||
| 		return untar(zr, dest, stripComponentLevel) | ||||
| 	} | ||||
|  | ||||
| 	return untar(file, dest, stripComponentLevel) | ||||
| } | ||||
|  | ||||
| func untar(r io.Reader, dir string, level int) (err error) { | ||||
| 	madeDir := map[string]bool{} | ||||
|  | ||||
| 	zr, err := gzip.NewReader(r) | ||||
| 	if err != nil { | ||||
| 		return fmt.Errorf("requires gzip-compressed body: %v", err) | ||||
| 	} | ||||
| 	tr := tar.NewReader(zr) | ||||
| 	tr := tar.NewReader(r) | ||||
| 	for { | ||||
| 		f, err := tr.Next() | ||||
| 		if err == io.EOF { | ||||
| @@ -252,7 +264,10 @@ func untar(r io.Reader, dir string, level int) (err error) { | ||||
| 		if err != nil { | ||||
| 			return fmt.Errorf("tar error: %v", err) | ||||
| 		} | ||||
| 		if !validRelPath(f.Name) { | ||||
| 		if strings.HasPrefix(f.Name, "/") { | ||||
| 			f.Name = fmt.Sprintf(".%s", f.Name) | ||||
| 		} | ||||
| 		if !validRelPath(f.Name) { // blocks path traversal attacks | ||||
| 			return fmt.Errorf("tar contained invalid name error %q", f.Name) | ||||
| 		} | ||||
| 		rel := filepath.FromSlash(f.Name) | ||||
| @@ -304,6 +319,10 @@ func untar(r io.Reader, dir string, level int) (err error) { | ||||
| 				return err | ||||
| 			} | ||||
| 			madeDir[abs] = true | ||||
| 		case mode&fs.ModeSymlink != 0: | ||||
| 			if err := os.Symlink(f.Linkname, abs); err != nil { | ||||
| 				return err | ||||
| 			} | ||||
| 		default: | ||||
| 			return fmt.Errorf("tar file entry %s contained unsupported file type %v", f.Name, mode) | ||||
| 		} | ||||
| @@ -311,6 +330,25 @@ func untar(r io.Reader, dir string, level int) (err error) { | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| // isFileGzipped checks the first 3 bytes of the given file to determine if it is gzipped or not. Returns `true` if the file is gzipped. | ||||
| func isFileGzipped(file string) (bool, error) { | ||||
| 	f, err := os.Open(file) | ||||
| 	defer f.Close() | ||||
|  | ||||
| 	if err != nil { | ||||
| 		return false, err | ||||
| 	} | ||||
|  | ||||
| 	b := make([]byte, 3) | ||||
| 	_, err = io.ReadFull(f, b) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		return false, err | ||||
| 	} | ||||
|  | ||||
| 	return b[0] == 0x1f && b[1] == 0x8b && b[2] == 8, nil | ||||
| } | ||||
|  | ||||
| func validRelPath(p string) bool { | ||||
| 	if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") { | ||||
| 		return false | ||||
| @@ -442,3 +480,13 @@ func (f Files) CurrentTime(format string) string { | ||||
| 	} | ||||
| 	return fmt.Sprint(time.Now().Format(fString)) | ||||
| } | ||||
|  | ||||
| // Open is a wrapper for os.Open | ||||
| func (f Files) Open(name string) (io.ReadWriteCloser, error) { | ||||
| 	return os.Open(name) | ||||
| } | ||||
|  | ||||
| // Create is a wrapper for os.Create | ||||
| func (f Files) Create(name string) (io.ReadWriteCloser, error) { | ||||
| 	return os.Create(name) | ||||
| } | ||||
|   | ||||
| @@ -126,6 +126,11 @@ func (pc *Protecode) SetOptions(options Options) { | ||||
| 	pc.client.SetOptions(httpOptions) | ||||
| } | ||||
|  | ||||
| //SetHttpClient setter function to set the http client | ||||
| func (pc *Protecode) SetHttpClient(client piperHttp.Uploader) { | ||||
| 	pc.client = client | ||||
| } | ||||
|  | ||||
| func (pc *Protecode) createURL(path string, pValue string, fParam string) string { | ||||
|  | ||||
| 	protecodeURL, err := url.Parse(pc.serverURL) | ||||
| @@ -339,7 +344,7 @@ func (pc *Protecode) UploadScanFile(cleanupMode, group, filePath, fileName, vers | ||||
| 	r, err := pc.client.UploadRequest(http.MethodPut, uploadURL, filePath, "file", headers, nil, "binary") | ||||
| 	if err != nil { | ||||
| 		//TODO: bubble up error | ||||
| 		pc.logger.WithError(err).Fatalf("Error during %v upload request", uploadURL) | ||||
| 		pc.logger.WithError(err).Fatalf("Error during upload request %v", uploadURL) | ||||
| 	} else { | ||||
| 		pc.logger.Info("Upload successful") | ||||
| 	} | ||||
|   | ||||
| @@ -4,7 +4,6 @@ import ( | ||||
| 	"crypto/sha1" | ||||
| 	"encoding/json" | ||||
| 	"fmt" | ||||
| 	"os" | ||||
| 	"path/filepath" | ||||
| 	"strings" | ||||
| 	"time" | ||||
| @@ -33,7 +32,7 @@ type ReportData struct { | ||||
| } | ||||
|  | ||||
| // WriteReport ... | ||||
| func WriteReport(data ReportData, reportPath string, reportFileName string, result map[string]int, writeToFile func(f string, d []byte, p os.FileMode) error) error { | ||||
| func WriteReport(data ReportData, reportPath string, reportFileName string, result map[string]int, fileUtils piperutils.FileUtils) error { | ||||
| 	data.Mandatory = true | ||||
| 	data.Count = fmt.Sprintf("%v", result["count"]) | ||||
| 	data.Cvss2GreaterOrEqualSeven = fmt.Sprintf("%v", result["cvss2GreaterOrEqualSeven"]) | ||||
| @@ -46,15 +45,15 @@ func WriteReport(data ReportData, reportPath string, reportFileName string, resu | ||||
| 		data.Count, data.Cvss2GreaterOrEqualSeven, data.Cvss3GreaterOrEqualSeven, | ||||
| 		data.ExcludedVulnerabilities, data.ExcludeCVEs, data.TriagedVulnerabilities, | ||||
| 		data.HistoricalVulnerabilities, data.Vulnerabilities) | ||||
| 	return writeJSON(reportPath, reportFileName, data, writeToFile) | ||||
| 	return writeJSON(reportPath, reportFileName, data, fileUtils) | ||||
| } | ||||
|  | ||||
| func writeJSON(path, name string, data interface{}, writeToFile func(f string, d []byte, p os.FileMode) error) error { | ||||
| func writeJSON(path, name string, data interface{}, fileUtils piperutils.FileUtils) error { | ||||
| 	jsonData, err := json.Marshal(data) | ||||
| 	if err != nil { | ||||
| 		return err | ||||
| 	} | ||||
| 	return writeToFile(filepath.Join(path, name), jsonData, 0644) | ||||
| 	return fileUtils.FileWrite(filepath.Join(path, name), jsonData, 0644) | ||||
| } | ||||
|  | ||||
| func CreateCustomReport(productName string, productID int, data map[string]int, vulns []Vuln) reporting.ScanReport { | ||||
| @@ -98,18 +97,17 @@ func CreateCustomReport(productName string, productID int, data map[string]int, | ||||
| 	return scanReport | ||||
| } | ||||
|  | ||||
| func WriteCustomReports(scanReport reporting.ScanReport, projectName, projectID string) ([]piperutils.Path, error) { | ||||
| 	utils := piperutils.Files{} | ||||
| func WriteCustomReports(scanReport reporting.ScanReport, projectName, projectID string, fileUtils piperutils.FileUtils) ([]piperutils.Path, error) { | ||||
| 	reportPaths := []piperutils.Path{} | ||||
|  | ||||
| 	// ignore templating errors since template is in our hands and issues will be detected with the automated tests | ||||
| 	htmlReport, _ := scanReport.ToHTML() | ||||
| 	htmlReportPath := filepath.Join(ReportsDirectory, "piper_protecode_report.html") | ||||
| 	// Ensure reporting directory exists | ||||
| 	if err := utils.MkdirAll(ReportsDirectory, 0777); err != nil { | ||||
| 	if err := fileUtils.MkdirAll(ReportsDirectory, 0777); err != nil { | ||||
| 		return reportPaths, errors.Wrapf(err, "failed to create report directory") | ||||
| 	} | ||||
| 	if err := utils.FileWrite(htmlReportPath, htmlReport, 0666); err != nil { | ||||
| 	if err := fileUtils.FileWrite(htmlReportPath, htmlReport, 0666); err != nil { | ||||
| 		log.SetErrorCategory(log.ErrorConfiguration) | ||||
| 		return reportPaths, errors.Wrapf(err, "failed to write html report") | ||||
| 	} | ||||
| @@ -118,13 +116,13 @@ func WriteCustomReports(scanReport reporting.ScanReport, projectName, projectID | ||||
| 	// JSON reports are used by step pipelineCreateSummary in order to e.g. prepare an issue creation in GitHub | ||||
| 	// ignore JSON errors since structure is in our hands | ||||
| 	jsonReport, _ := scanReport.ToJSON() | ||||
| 	if exists, _ := utils.DirExists(reporting.StepReportDirectory); !exists { | ||||
| 		err := utils.MkdirAll(reporting.StepReportDirectory, 0777) | ||||
| 	if exists, _ := fileUtils.DirExists(reporting.StepReportDirectory); !exists { | ||||
| 		err := fileUtils.MkdirAll(reporting.StepReportDirectory, 0777) | ||||
| 		if err != nil { | ||||
| 			return reportPaths, errors.Wrap(err, "failed to create reporting directory") | ||||
| 		} | ||||
| 	} | ||||
| 	if err := utils.FileWrite(filepath.Join(reporting.StepReportDirectory, fmt.Sprintf("protecodeExecuteScan_osvm_%v.json", reportShaProtecode([]string{projectName, projectID}))), jsonReport, 0666); err != nil { | ||||
| 	if err := fileUtils.FileWrite(filepath.Join(reporting.StepReportDirectory, fmt.Sprintf("protecodeExecuteScan_osvm_%v.json", reportShaProtecode([]string{projectName, projectID}))), jsonReport, 0666); err != nil { | ||||
| 		return reportPaths, errors.Wrapf(err, "failed to write json report") | ||||
| 	} | ||||
| 	// we do not add the json report to the overall list of reports for now, | ||||
|   | ||||
| @@ -2,20 +2,15 @@ package protecode | ||||
|  | ||||
| import ( | ||||
| 	"fmt" | ||||
| 	"os" | ||||
| 	"testing" | ||||
|  | ||||
| 	"github.com/SAP/jenkins-library/pkg/mock" | ||||
| 	"github.com/stretchr/testify/assert" | ||||
| ) | ||||
|  | ||||
| var fileContent string | ||||
|  | ||||
| func writeToFileMock(f string, d []byte, p os.FileMode) error { | ||||
| 	fileContent = string(d) | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| func TestWriteReport(t *testing.T) { | ||||
| 	files := mock.FilesMock{} | ||||
|  | ||||
| 	expected := "{\"target\":\"REPORTFILENAME\",\"mandatory\":true,\"productID\":\"4711\",\"serverUrl\":\"DUMMYURL\",\"count\":\"0\",\"cvss2GreaterOrEqualSeven\":\"4\",\"cvss3GreaterOrEqualSeven\":\"3\",\"excludedVulnerabilities\":\"2\",\"triagedVulnerabilities\":\"0\",\"historicalVulnerabilities\":\"1\",\"Vulnerabilities\":[{\"cve\":\"Vulnerability\",\"cvss\":2.5,\"cvss3_score\":\"5.5\"}]}" | ||||
|  | ||||
| 	var parsedResult map[string]int = make(map[string]int) | ||||
| @@ -25,7 +20,11 @@ func TestWriteReport(t *testing.T) { | ||||
| 	parsedResult["cvss2GreaterOrEqualSeven"] = 4 | ||||
| 	parsedResult["vulnerabilities"] = 5 | ||||
|  | ||||
| 	err := WriteReport(ReportData{ServerURL: "DUMMYURL", FailOnSevereVulnerabilities: false, ExcludeCVEs: "", Target: "REPORTFILENAME", ProductID: fmt.Sprintf("%v", 4711), Vulnerabilities: []Vuln{{"Vulnerability", 2.5, "5.5"}}}, ".", "", parsedResult, writeToFileMock) | ||||
| 	assert.Equal(t, fileContent, expected, "content should be not empty") | ||||
| 	assert.NoError(t, err) | ||||
| 	err := WriteReport(ReportData{ServerURL: "DUMMYURL", FailOnSevereVulnerabilities: false, ExcludeCVEs: "", Target: "REPORTFILENAME", ProductID: fmt.Sprintf("%v", 4711), Vulnerabilities: []Vuln{{"Vulnerability", 2.5, "5.5"}}}, ".", "report.json", parsedResult, &files) | ||||
|  | ||||
| 	if assert.NoError(t, err) { | ||||
| 		content, err := files.FileRead("report.json") | ||||
| 		assert.NoError(t, err) | ||||
| 		assert.Equal(t, expected, string(content)) | ||||
| 	} | ||||
| } | ||||
|   | ||||
| @@ -19,7 +19,7 @@ spec: | ||||
|         aliases: | ||||
|           - name: dockerRegistryUrl | ||||
|         type: string | ||||
|         description: The reference to the container registry where the image is located. | ||||
|         description: "For `buildTool: docker`: Url of the container registry - typically provided by the CI/CD environment." | ||||
|         mandatory: true | ||||
|         resourceRef: | ||||
|           - name: commonPipelineEnvironment | ||||
| @@ -72,14 +72,7 @@ spec: | ||||
|             param: custom/repositoryUsername | ||||
|       - name: filePath | ||||
|         type: string | ||||
|         description: The path to the file to which the image should be saved. Defaults to `containerImage.tar` | ||||
|         scope: | ||||
|           - PARAMETERS | ||||
|           - STAGES | ||||
|           - STEPS | ||||
|       - name: includeLayers | ||||
|         type: bool | ||||
|         description: Flag if the docker layers should be included | ||||
|         description: The path to the file to which the image should be saved. | ||||
|         scope: | ||||
|           - PARAMETERS | ||||
|           - STAGES | ||||
|   | ||||
| @@ -114,14 +114,6 @@ spec: | ||||
|           - PARAMETERS | ||||
|           - STAGES | ||||
|           - STEPS | ||||
|       - name: scanImageIncludeLayers | ||||
|         type: bool | ||||
|         description: "For `buildTool: docker`: Defines if layers should be included." | ||||
|         scope: | ||||
|           - PARAMETERS | ||||
|           - STAGES | ||||
|           - STEPS | ||||
|         default: true | ||||
|       - name: scanImageRegistryUrl | ||||
|         type: string | ||||
|         description: "For `buildTool: docker`: Defines the registry where the scanImage is located." | ||||
|   | ||||
| @@ -99,13 +99,6 @@ spec: | ||||
|           - PARAMETERS | ||||
|           - STAGES | ||||
|           - STEPS | ||||
|       - name: includeLayers | ||||
|         type: bool | ||||
|         description: Flag if the docker layers should be included | ||||
|         scope: | ||||
|           - PARAMETERS | ||||
|           - STAGES | ||||
|           - STEPS | ||||
|       - name: timeoutMinutes | ||||
|         aliases: | ||||
|           - name: protecodeTimeoutMinutes | ||||
|   | ||||
| @@ -353,14 +353,6 @@ spec: | ||||
|           - PARAMETERS | ||||
|           - STAGES | ||||
|           - STEPS | ||||
|       - name: scanImageIncludeLayers | ||||
|         type: bool | ||||
|         description: "For `buildTool: docker`: Defines if layers should be included." | ||||
|         scope: | ||||
|           - PARAMETERS | ||||
|           - STAGES | ||||
|           - STEPS | ||||
|         default: true | ||||
|       - name: scanImageRegistryUrl | ||||
|         type: string | ||||
|         description: "For `buildTool: docker`: Defines the registry where the scanImage is located." | ||||
|   | ||||
		Reference in New Issue
	
	Block a user