Cnb build custom buildpacks (#3090)

* [WIP] cnbBuild custom buildpacks draft

Co-authored-by: Pavel Busko <pavel.busko@sap.com>

* Store custom buildpacks in the dedicated tmp folder

Co-authored-by: Ralf Pannemans <ralf.pannemans@sap.com>

* added test

Co-authored-by: Pavel Busko <pavel.busko@sap.com>

* updated documentation

Co-authored-by: Ralf Pannemans <ralf.pannemans@sap.com>

* use Files for toml files cleanup

Co-authored-by: Pavel Busko <pavel.busko@sap.com>

* Add missing function to the FileUtils interface

Co-authored-by: Ralf Pannemans <ralf.pannemans@sap.com>

Co-authored-by: Ralf Pannemans <ralf.pannemans@sap.com>

cmd/cnbBuild.go

@@ -8,6 +8,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/SAP/jenkins-library/pkg/cnbutils"
 	"github.com/SAP/jenkins-library/pkg/command"
 	"github.com/SAP/jenkins-library/pkg/docker"
 	"github.com/SAP/jenkins-library/pkg/log"
@@ -23,27 +24,33 @@ const (
 	exporterPath = "/cnb/lifecycle/exporter"
 )
 
-type cnbBuildUtils interface {
-	command.ExecRunner
-
-	FileExists(filename string) (bool, error)
-	FileRead(path string) ([]byte, error)
-	FileWrite(path string, content []byte, perm os.FileMode) error
-	MkdirAll(path string, perm os.FileMode) error
-	Getwd() (string, error)
-	Glob(pattern string) (matches []string, err error)
-	Copy(src, dest string) (int64, error)
-}
-
 type cnbBuildUtilsBundle struct {
 	*command.Command
 	*piperutils.Files
+	*docker.Client
 }
 
-func newCnbBuildUtils() cnbBuildUtils {
+func setCustomBuildpacks(bpacks []string, utils cnbutils.BuildUtils) (string, string, error) {
+	buildpacksPath := "/tmp/buildpacks"
+	orderPath := "/tmp/buildpacks/order.toml"
+	newOrder, err := cnbutils.DownloadBuildpacks(buildpacksPath, bpacks, utils)
+	if err != nil {
+		return "", "", err
+	}
+
+	err = newOrder.Save(orderPath)
+	if err != nil {
+		return "", "", err
+	}
+
+	return buildpacksPath, orderPath, nil
+}
+
+func newCnbBuildUtils() cnbutils.BuildUtils {
 	utils := cnbBuildUtilsBundle{
 		Command: &command.Command{},
 		Files:   &piperutils.Files{},
+		Client:  &docker.Client{},
 	}
 	utils.Stdout(log.Writer())
 	utils.Stderr(log.Writer())
@@ -71,7 +78,7 @@ func isDir(path string) (bool, error) {
 	return info.IsDir(), nil
 }
 
-func isBuilder(utils cnbBuildUtils) (bool, error) {
+func isBuilder(utils cnbutils.BuildUtils) (bool, error) {
 	for _, path := range []string{detectorPath, builderPath, exporterPath} {
 		exists, err := utils.FileExists(path)
 		if err != nil || !exists {
@@ -81,7 +88,7 @@ func isBuilder(utils cnbBuildUtils) (bool, error) {
 	return true, nil
 }
 
-func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, utils cnbBuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment) error {
+func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment) error {
 	var err error
 
 	exists, err := isBuilder(utils)
@@ -163,6 +170,20 @@ func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, u
 		}
 	}
 
+	var buildpacksPath = "/cnb/buildpacks"
+	var orderPath = "/cnb/order.toml"
+
+	if config.Buildpacks != nil && len(config.Buildpacks) != 0 {
+		log.Entry().Infof("Setting custom buildpacks: '%v'", config.Buildpacks)
+		buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, utils)
+		defer utils.RemoveAll(buildpacksPath)
+		defer utils.RemoveAll(orderPath)
+		if err != nil {
+			log.SetErrorCategory(log.ErrorBuild)
+			return errors.Wrapf(err, "Setting custom buildpacks: %v", config.Buildpacks)
+		}
+	}
+
 	var containerImage string
 	var containerImageTag string
 
@@ -187,13 +208,13 @@ func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, u
 		return errors.New("containerRegistryUrl, containerImageName and containerImageTag must be present")
 	}
 
-	err = utils.RunExecutable(detectorPath)
+	err = utils.RunExecutable(detectorPath, "-buildpacks", buildpacksPath, "-order", orderPath)
 	if err != nil {
 		log.SetErrorCategory(log.ErrorBuild)
 		return errors.Wrap(err, fmt.Sprintf("execution of '%s' failed", detectorPath))
 	}
 
-	err = utils.RunExecutable(builderPath)
+	err = utils.RunExecutable(builderPath, "-buildpacks", buildpacksPath)
 	if err != nil {
 		log.SetErrorCategory(log.ErrorBuild)
 		return errors.Wrap(err, fmt.Sprintf("execution of '%s' failed", builderPath))

cmd/cnbBuild_generated.go

@@ -17,11 +17,12 @@ import (
 )
 
 type cnbBuildOptions struct {
-	ContainerImageName   string `json:"containerImageName,omitempty"`
-	ContainerImageTag    string `json:"containerImageTag,omitempty"`
-	ContainerRegistryURL string `json:"containerRegistryUrl,omitempty"`
-	Path                 string `json:"path,omitempty"`
-	DockerConfigJSON     string `json:"dockerConfigJSON,omitempty"`
+	ContainerImageName   string   `json:"containerImageName,omitempty"`
+	ContainerImageTag    string   `json:"containerImageTag,omitempty"`
+	ContainerRegistryURL string   `json:"containerRegistryUrl,omitempty"`
+	Buildpacks           []string `json:"buildpacks,omitempty"`
+	Path                 string   `json:"path,omitempty"`
+	DockerConfigJSON     string   `json:"dockerConfigJSON,omitempty"`
 }
 
 type cnbBuildCommonPipelineEnvironment struct {
@@ -135,6 +136,7 @@ func addCnbBuildFlags(cmd *cobra.Command, stepConfig *cnbBuildOptions) {
 	cmd.Flags().StringVar(&stepConfig.ContainerImageName, "containerImageName", os.Getenv("PIPER_containerImageName"), "Name of the container which will be built")
 	cmd.Flags().StringVar(&stepConfig.ContainerImageTag, "containerImageTag", os.Getenv("PIPER_containerImageTag"), "Tag of the container which will be built")
 	cmd.Flags().StringVar(&stepConfig.ContainerRegistryURL, "containerRegistryUrl", os.Getenv("PIPER_containerRegistryUrl"), "Container registry where the image should be pushed to")
+	cmd.Flags().StringSliceVar(&stepConfig.Buildpacks, "buildpacks", []string{}, "List of custom buildpacks to use in the form of '<hostname>/<repo>[:<tag>]'.")
 	cmd.Flags().StringVar(&stepConfig.Path, "path", os.Getenv("PIPER_path"), "The path should either point to your sources or an artifact build before.")
 	cmd.Flags().StringVar(&stepConfig.DockerConfigJSON, "dockerConfigJSON", os.Getenv("PIPER_dockerConfigJSON"), "Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).")
 
@@ -195,6 +197,15 @@ func cnbBuildMetadata() config.StepData {
 						Aliases:   []config.Alias{{Name: "dockerRegistryUrl"}},
 						Default:   os.Getenv("PIPER_containerRegistryUrl"),
 					},
+					{
+						Name:        "buildpacks",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "[]string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     []string{},
+					},
 					{
 						Name:        "path",
 						ResourceRef: []config.ResourceReference{},

cmd/cnbBuild_test.go

@@ -4,25 +4,22 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/SAP/jenkins-library/pkg/cnbutils"
 	"github.com/SAP/jenkins-library/pkg/mock"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
 	"github.com/stretchr/testify/assert"
 )
 
-type cnbBuildMockUtils struct {
-	*mock.ExecMockRunner
-	*mock.FilesMock
-}
-
-func newCnbBuildTestsUtils() cnbBuildMockUtils {
-	utils := cnbBuildMockUtils{
+func newCnbBuildTestsUtils() cnbutils.MockUtils {
+	utils := cnbutils.MockUtils{
 		ExecMockRunner: &mock.ExecMockRunner{},
 		FilesMock:      &mock.FilesMock{},
+		DockerMock:     &cnbutils.DockerMock{},
 	}
 	return utils
 }
 
-func addBuilderFiles(utils *cnbBuildMockUtils) {
+func addBuilderFiles(utils *cnbutils.MockUtils) {
 	for _, path := range []string{detectorPath, builderPath, exporterPath} {
 		utils.FilesMock.AddFile(path, []byte(`xyz`))
 	}
@@ -47,7 +44,7 @@ func TestRunCnbBuild(t *testing.T) {
 		utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
 		addBuilderFiles(&utils)
 
-		err := runCnbBuild(&config, &telemetry.CustomData{}, utils, &commonPipelineEnvironment)
+		err := runCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment)
 
 		assert.NoError(t, err)
 		runner := utils.ExecMockRunner
@@ -55,6 +52,8 @@ func TestRunCnbBuild(t *testing.T) {
 		assert.Equal(t, "/cnb/lifecycle/detector", runner.Calls[0].Exec)
 		assert.Equal(t, "/cnb/lifecycle/builder", runner.Calls[1].Exec)
 		assert.Equal(t, "/cnb/lifecycle/exporter", runner.Calls[2].Exec)
+		assert.Equal(t, []string{"-buildpacks", "/cnb/buildpacks", "-order", "/cnb/order.toml"}, runner.Calls[0].Params)
+		assert.Equal(t, []string{"-buildpacks", "/cnb/buildpacks"}, runner.Calls[1].Params)
 		assert.Equal(t, []string{fmt.Sprintf("%s/%s:%s", registry, config.ContainerImageName, config.ContainerImageTag), fmt.Sprintf("%s/%s:latest", registry, config.ContainerImageName)}, runner.Calls[2].Params)
 	})
 
@@ -72,7 +71,7 @@ func TestRunCnbBuild(t *testing.T) {
 		utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
 		addBuilderFiles(&utils)
 
-		err := runCnbBuild(&config, &telemetry.CustomData{}, utils, &commonPipelineEnvironment)
+		err := runCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment)
 
 		assert.NoError(t, err)
 		runner := utils.ExecMockRunner
@@ -80,6 +79,36 @@ func TestRunCnbBuild(t *testing.T) {
 		assert.Equal(t, "/cnb/lifecycle/detector", runner.Calls[0].Exec)
 		assert.Equal(t, "/cnb/lifecycle/builder", runner.Calls[1].Exec)
 		assert.Equal(t, "/cnb/lifecycle/exporter", runner.Calls[2].Exec)
+		assert.Equal(t, []string{"-buildpacks", "/cnb/buildpacks", "-order", "/cnb/order.toml"}, runner.Calls[0].Params)
+		assert.Equal(t, []string{"-buildpacks", "/cnb/buildpacks"}, runner.Calls[1].Params)
+		assert.Equal(t, []string{fmt.Sprintf("%s/%s:%s", registry, config.ContainerImageName, config.ContainerImageTag), fmt.Sprintf("%s/%s:latest", registry, config.ContainerImageName)}, runner.Calls[2].Params)
+	})
+
+	t.Run("success case (custom buildpacks)", func(t *testing.T) {
+		t.Parallel()
+		registry := "some-registry"
+		config := cnbBuildOptions{
+			ContainerImageName:   "my-image",
+			ContainerImageTag:    "0.0.1",
+			ContainerRegistryURL: registry,
+			DockerConfigJSON:     "/path/to/config.json",
+			Buildpacks:           []string{"test"},
+		}
+
+		utils := newCnbBuildTestsUtils()
+		utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
+		addBuilderFiles(&utils)
+
+		err := runCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment)
+
+		assert.NoError(t, err)
+		runner := utils.ExecMockRunner
+		assert.Contains(t, runner.Env, "CNB_REGISTRY_AUTH={\"my-registry\":\"Basic dXNlcjpwYXNz\"}")
+		assert.Equal(t, "/cnb/lifecycle/detector", runner.Calls[0].Exec)
+		assert.Equal(t, "/cnb/lifecycle/builder", runner.Calls[1].Exec)
+		assert.Equal(t, "/cnb/lifecycle/exporter", runner.Calls[2].Exec)
+		assert.Equal(t, []string{"-buildpacks", "/tmp/buildpacks", "-order", "/tmp/buildpacks/order.toml"}, runner.Calls[0].Params)
+		assert.Equal(t, []string{"-buildpacks", "/tmp/buildpacks"}, runner.Calls[1].Params)
 		assert.Equal(t, []string{fmt.Sprintf("%s/%s:%s", registry, config.ContainerImageName, config.ContainerImageTag), fmt.Sprintf("%s/%s:latest", registry, config.ContainerImageName)}, runner.Calls[2].Params)
 	})
 
@@ -94,7 +123,7 @@ func TestRunCnbBuild(t *testing.T) {
 		utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":"dXNlcjpwYXNz"}}`))
 		addBuilderFiles(&utils)
 
-		err := runCnbBuild(&config, nil, utils, &commonPipelineEnvironment)
+		err := runCnbBuild(&config, nil, &utils, &commonPipelineEnvironment)
 		assert.EqualError(t, err, "failed to parse DockerConfigJSON file '/path/to/config.json': json: cannot unmarshal string into Go struct field ConfigFile.auths of type types.AuthConfig")
 	})
 
@@ -108,7 +137,7 @@ func TestRunCnbBuild(t *testing.T) {
 		utils := newCnbBuildTestsUtils()
 		addBuilderFiles(&utils)
 
-		err := runCnbBuild(&config, nil, utils, &commonPipelineEnvironment)
+		err := runCnbBuild(&config, nil, &utils, &commonPipelineEnvironment)
 		assert.EqualError(t, err, "failed to read DockerConfigJSON file 'not-there': could not read 'not-there'")
 	})
 
@@ -118,7 +147,7 @@ func TestRunCnbBuild(t *testing.T) {
 
 		utils := newCnbBuildTestsUtils()
 
-		err := runCnbBuild(&config, nil, utils, &commonPipelineEnvironment)
+		err := runCnbBuild(&config, nil, &utils, &commonPipelineEnvironment)
 		assert.EqualError(t, err, "the provided dockerImage is not a valid builder")
 	})
 }

cmd/kanikoExecute_test.go

@@ -37,20 +37,13 @@ func (c *kanikoMockClient) SendRequest(method, url string, body io.Reader, heade
 }
 
 type kanikoFileMock struct {
+	*mock.FilesMock
 	fileReadContent  map[string]string
 	fileReadErr      map[string]error
 	fileWriteContent map[string]string
 	fileWriteErr     map[string]error
 }
 
-func (f *kanikoFileMock) FileExists(path string) (bool, error) {
-	return true, nil
-}
-
-func (f *kanikoFileMock) Copy(src, dest string) (int64, error) {
-	return 0, nil
-}
-
 func (f *kanikoFileMock) FileRead(path string) ([]byte, error) {
 	if f.fileReadErr[path] != nil {
 		return []byte{}, f.fileReadErr[path]
@@ -66,26 +59,6 @@ func (f *kanikoFileMock) FileWrite(path string, content []byte, perm os.FileMode
 	return nil
 }
 
-func (f *kanikoFileMock) MkdirAll(path string, perm os.FileMode) error {
-	return nil
-}
-
-func (f *kanikoFileMock) Chmod(path string, mode os.FileMode) error {
-	return fmt.Errorf("not implemented. func is only present in order to fullfil the interface contract. Needs to be ajusted in case it gets used.")
-}
-
-func (f *kanikoFileMock) Abs(path string) (string, error) {
-	return "", fmt.Errorf("not implemented. func is only present in order to fullfil the interface contract. Needs to be ajusted in case it gets used.")
-}
-
-func (f *kanikoFileMock) Glob(pattern string) (matches []string, err error) {
-	return nil, fmt.Errorf("not implemented. func is only present in order to fullfil the interface contract. Needs to be ajusted in case it gets used.")
-}
-
-func (f *kanikoFileMock) Chdir(pattern string) error {
-	return nil
-}
-
 func TestRunKanikoExecute(t *testing.T) {
 
 	commonPipelineEnvironment := kanikoExecuteCommonPipelineEnvironment{}

cmd/xsDeploy_test.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"os"
 	"sync"
 	"testing"
 
@@ -15,6 +14,7 @@ import (
 )
 
 type FileUtilsMock struct {
+	*mock.FilesMock
 	copiedFiles []string
 }
 
@@ -27,34 +27,6 @@ func (f *FileUtilsMock) Copy(src, dest string) (int64, error) {
 	return 0, nil
 }
 
-func (f *FileUtilsMock) FileRead(path string) ([]byte, error) {
-	return []byte{}, nil
-}
-
-func (f *FileUtilsMock) FileWrite(path string, content []byte, perm os.FileMode) error {
-	return nil
-}
-
-func (f *FileUtilsMock) MkdirAll(path string, perm os.FileMode) error {
-	return nil
-}
-
-func (f *FileUtilsMock) Chmod(path string, mode os.FileMode) error {
-	return fmt.Errorf("not implemented. func is only present in order to fullfil the interface contract. Needs to be ajusted in case it gets used.")
-}
-
-func (f *FileUtilsMock) Abs(path string) (string, error) {
-	return "", fmt.Errorf("not implemented. func is only present in order to fullfil the interface contract. Needs to be ajusted in case it gets used.")
-}
-
-func (f *FileUtilsMock) Glob(pattern string) (matches []string, err error) {
-	return nil, fmt.Errorf("not implemented. func is only present in order to fullfil the interface contract. Needs to be ajusted in case it gets used.")
-}
-
-func (f *FileUtilsMock) Chdir(pattern string) error {
-	return nil
-}
-
 func TestDeploy(t *testing.T) {
 	myXsDeployOptions := xsDeployOptions{
 		APIURL:                "https://example.org:12345",

documentation/docs/steps/cnbBuild.md

@@ -37,3 +37,16 @@ cnbBuild(
     containerImageRegistryUrl: 'gcr.io'
 )
 ```
+
+## Example 3: User provided buildpacks
+
+```groovy
+cnbBuild(
+    script: script,
+    dockerConfigJsonCredentialsId: 'DOCKER_REGISTRY_CREDS',
+    containerImageName: 'images/example',
+    containerImageTag: 'v0.0.1',
+    containerImageRegistryUrl: 'gcr.io',
+    buildpacks: ['gcr.io/paketo-buildpacks/nodejs', 'paketo-community/build-plan']
+)
+```

go.mod

@@ -41,6 +41,7 @@ require (
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5
 	github.com/motemen/go-nuts v0.0.0-20200601065735-3df31f16cb2f
+	github.com/pelletier/go-toml v1.9.3
 	github.com/piper-validation/fortify-client-go v0.0.0-20210114140201-1261216783c6
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.7.0

go.sum

@@ -1282,6 +1282,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
+github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
+github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ=

integration/integration_cnb_test.go

@@ -23,6 +23,42 @@ func TestNpmProject(t *testing.T) {
 	container.assertHasOutput(t, "failed to write image to the following tags: [test/not-found:0.0.1")
 }
 
+func TestNpmCustomBuildpacksFullProject(t *testing.T) {
+	t.Parallel()
+	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
+		Image:   "paketobuildpacks/builder:full",
+		User:    "cnb",
+		TestDir: []string{"testdata", "TestMtaIntegration", "npm"},
+	})
+
+	container.whenRunningPiperCommand("cnbBuild", "--buildpacks", "gcr.io/paketo-buildpacks/nodejs", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", "test")
+
+	container.assertHasOutput(t, "Setting custom buildpacks: '[gcr.io/paketo-buildpacks/nodejs]'")
+	container.assertHasOutput(t, "Downloading buildpack 'gcr.io/paketo-buildpacks/nodejs' to /tmp/nodejs")
+	container.assertHasOutput(t, "running command: /cnb/lifecycle/detector")
+	container.assertHasOutput(t, "Paketo NPM Start Buildpack")
+	container.assertHasOutput(t, "Saving test/not-found:0.0.1")
+	container.assertHasOutput(t, "failed to write image to the following tags: [test/not-found:0.0.1")
+}
+
+func TestNpmCustomBuildpacksBuildpacklessProject(t *testing.T) {
+	t.Parallel()
+	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
+		Image:   "paketobuildpacks/builder:buildpackless-full",
+		User:    "cnb",
+		TestDir: []string{"testdata", "TestMtaIntegration", "npm"},
+	})
+
+	container.whenRunningPiperCommand("cnbBuild", "--buildpacks", "gcr.io/paketo-buildpacks/nodejs", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", "test")
+
+	container.assertHasOutput(t, "Setting custom buildpacks: '[gcr.io/paketo-buildpacks/nodejs]'")
+	container.assertHasOutput(t, "Downloading buildpack 'gcr.io/paketo-buildpacks/nodejs' to /tmp/nodejs")
+	container.assertHasOutput(t, "running command: /cnb/lifecycle/detector")
+	container.assertHasOutput(t, "Paketo NPM Start Buildpack")
+	container.assertHasOutput(t, "Saving test/not-found:0.0.1")
+	container.assertHasOutput(t, "failed to write image to the following tags: [test/not-found:0.0.1")
+}
+
 func TestWrongBuilderProject(t *testing.T) {
 	t.Parallel()
 	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{

pkg/cnbutils/buildpack.go

@@ -0,0 +1,106 @@
+// Package cnbutils provides utility functions to interact with Buildpacks
+package cnbutils
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/SAP/jenkins-library/pkg/log"
+)
+
+type BuildPackMetadata struct {
+	ID          string    `json:"id,omitempty"`
+	Name        string    `json:"name,omitempty"`
+	Version     string    `json:"version,omitempty"`
+	Description string    `json:"description,omitempty"`
+	Homepage    string    `json:"homepage,omitempty"`
+	Keywords    []string  `json:"keywords,omitempty"`
+	Licenses    []License `json:"licenses,omitempty"`
+}
+
+type License struct {
+	Type string `json:"type"`
+	URI  string `json:"uri"`
+}
+
+func DownloadBuildpacks(path string, bpacks []string, utils BuildUtils) (Order, error) {
+	var order Order
+	for _, bpack := range bpacks {
+		var bpackMeta BuildPackMetadata
+		tempDir, err := utils.TempDir("", filepath.Base(bpack))
+		if err != nil {
+			return Order{}, fmt.Errorf("failed to create temp directory, error: %s", err.Error())
+		}
+		defer utils.RemoveAll(tempDir)
+
+		log.Entry().Infof("Downloading buildpack '%s' to %s", bpack, tempDir)
+		img, err := utils.DownloadImageToPath(bpack, tempDir)
+		if err != nil {
+			return Order{}, fmt.Errorf("failed download buildpack image '%s', error: %s", bpack, err.Error())
+		}
+
+		imgConf, err := img.Image.ConfigFile()
+		if err != nil {
+			return Order{}, fmt.Errorf("failed to read '%s' image config, error: %s", bpack, err.Error())
+		}
+
+		err = json.Unmarshal([]byte(imgConf.Config.Labels["io.buildpacks.buildpackage.metadata"]), &bpackMeta)
+		if err != nil {
+			return Order{}, fmt.Errorf("failed unmarshal '%s' image label, error: %s", bpack, err.Error())
+		}
+		log.Entry().Debugf("Buildpack metadata: '%v'", bpackMeta)
+		order.Order = append(order.Order, OrderEntry{
+			Group: []BuildpackRef{{
+				ID:       bpackMeta.ID,
+				Version:  bpackMeta.Version,
+				Optional: false,
+			}},
+		})
+
+		err = copyBuildPack(filepath.Join(tempDir, "cnb/buildpacks"), path, utils)
+		if err != nil {
+			return Order{}, err
+		}
+	}
+
+	order.Utils = utils
+
+	return order, nil
+}
+
+func copyBuildPack(src, dst string, utils BuildUtils) error {
+	buildpacks, err := utils.Glob(filepath.Join(src, "*"))
+	if err != nil {
+		return fmt.Errorf("failed to read directory: %s, error: %s", src, err.Error())
+	}
+
+	for _, buildpack := range buildpacks {
+		versions, err := utils.Glob(filepath.Join(buildpack, "*"))
+		if err != nil {
+			return fmt.Errorf("failed to read directory: %s, error: %s", buildpack, err.Error())
+		}
+		for _, srcVersionPath := range versions {
+			destVersionPath := filepath.Join(dst, strings.ReplaceAll(srcVersionPath, src, ""))
+
+			exists, err := utils.FileExists(destVersionPath)
+			if err != nil {
+				return fmt.Errorf("failed to check if directory exists: '%s', error: '%s'", destVersionPath, err.Error())
+			}
+			if exists {
+				utils.RemoveAll(destVersionPath)
+			}
+
+			if err := utils.MkdirAll(filepath.Dir(destVersionPath), 0755); err != nil {
+				return fmt.Errorf("failed to create directory: '%s', error: '%s'", filepath.Dir(destVersionPath), err.Error())
+			}
+
+			err = utils.FileRename(srcVersionPath, destVersionPath)
+			if err != nil {
+				return fmt.Errorf("failed to move '%s' to '%s', error: %s", srcVersionPath, destVersionPath, err.Error())
+			}
+		}
+	}
+	return nil
+}

pkg/cnbutils/buildpack_test.go

@@ -0,0 +1,35 @@
+package cnbutils
+
+import (
+	"testing"
+
+	"github.com/SAP/jenkins-library/pkg/mock"
+	"github.com/stretchr/testify/assert"
+)
+
+var mockUtils = MockUtils{
+	ExecMockRunner: &mock.ExecMockRunner{},
+	FilesMock:      &mock.FilesMock{},
+	DockerMock:     &DockerMock{},
+}
+
+func TestBuildpackDownload(t *testing.T) {
+	t.Run("successfully downloads a buildpack", func(t *testing.T) {
+		mockUtils.AddDir("/tmp/testtest")
+		_, err := DownloadBuildpacks("/test", []string{"test"}, mockUtils)
+
+		assert.NoError(t, err)
+		assert.True(t, mockUtils.HasRemovedFile("/tmp/testtest"))
+	})
+}
+
+func TestBuildpackCopy(t *testing.T) {
+	t.Run("successfully downloads a buildpack", func(t *testing.T) {
+
+		mockUtils.AddDir("/src/buildpack/0.0.1")
+		mockUtils.AddDir("/dst")
+		err := copyBuildPack("/src", "/dst", mockUtils)
+
+		assert.NoError(t, err)
+	})
+}

pkg/cnbutils/mock.go

@@ -0,0 +1,54 @@
+// +build !release
+
+package cnbutils
+
+import (
+	"io"
+
+	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
+	"github.com/SAP/jenkins-library/pkg/docker"
+	"github.com/SAP/jenkins-library/pkg/mock"
+	"github.com/SAP/jenkins-library/pkg/piperutils"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	fakeImage "github.com/google/go-containerregistry/pkg/v1/fake"
+)
+
+type MockUtils struct {
+	*mock.ExecMockRunner
+	*mock.FilesMock
+	*DockerMock
+}
+
+func (c *MockUtils) GetDockerClient() docker.Download {
+	return c.DockerMock
+}
+
+func (c *MockUtils) GetFileUtils() piperutils.FileUtils {
+	return c.FilesMock
+}
+
+type DockerMock struct{}
+
+func (d *DockerMock) DownloadImageToPath(_, filePath string) (pkgutil.Image, error) {
+
+	fakeImage := fakeImage.FakeImage{}
+	fakeImage.ConfigFileReturns(&v1.ConfigFile{
+		Config: v1.Config{
+			Labels: map[string]string{
+				"io.buildpacks.buildpackage.metadata": "{\"id\": \"testbuildpack\", \"version\": \"0.0.1\"}",
+			},
+		},
+	}, nil)
+	img := pkgutil.Image{
+		Image: &fakeImage,
+	}
+	return img, nil
+}
+
+func (d *DockerMock) GetImageSource() (string, error) {
+	return "imageSource", nil
+}
+
+func (d *DockerMock) TarImage(writer io.Writer, image pkgutil.Image) error {
+	return nil
+}

pkg/cnbutils/order.go

@@ -0,0 +1,38 @@
+package cnbutils
+
+import (
+	"bytes"
+
+	"github.com/pelletier/go-toml"
+)
+
+type Order struct {
+	Order []OrderEntry `toml:"order"`
+	Utils BuildUtils   `toml:"-"`
+}
+
+type OrderEntry struct {
+	Group []BuildpackRef `toml:"group" json:"group"`
+}
+
+type BuildpackRef struct {
+	ID       string `toml:"id"`
+	Version  string `toml:"version"`
+	Optional bool   `toml:"optional,omitempty" json:"optional,omitempty" yaml:"optional,omitempty"`
+}
+
+func (o Order) Save(path string) error {
+	var buf bytes.Buffer
+
+	err := toml.NewEncoder(&buf).Encode(o)
+	if err != nil {
+		return err
+	}
+
+	err = o.Utils.FileWrite(path, buf.Bytes(), 0644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/cnbutils/order_test.go

@@ -0,0 +1,63 @@
+package cnbutils
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/SAP/jenkins-library/pkg/mock"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestOrderSave(t *testing.T) {
+	t.Run("successfully Encode struct to toml format", func(t *testing.T) {
+		mockUtils := MockUtils{
+			ExecMockRunner: &mock.ExecMockRunner{},
+			FilesMock:      &mock.FilesMock{},
+			DockerMock:     &DockerMock{},
+		}
+		testOrder := Order{
+			Order: []OrderEntry{{
+				Group: []BuildpackRef{{
+					ID:       "test",
+					Version:  "0.0.1",
+					Optional: true,
+				}},
+			}},
+			Utils: mockUtils,
+		}
+
+		err := testOrder.Save("/tmp/order.toml")
+
+		assert.NoError(t, err)
+		assert.True(t, mockUtils.HasWrittenFile("/tmp/order.toml"))
+		result, err := mockUtils.FileRead("/tmp/order.toml")
+		assert.NoError(t, err)
+		assert.Equal(t, "\n[[order]]\n\n  [[order.group]]\n    id = \"test\"\n    optional = true\n    version = \"0.0.1\"\n", string(result))
+	})
+
+	t.Run("raises an error if unable to write the file", func(t *testing.T) {
+		mockUtils := MockUtils{
+			ExecMockRunner: &mock.ExecMockRunner{},
+			FilesMock:      &mock.FilesMock{},
+			DockerMock:     &DockerMock{},
+		}
+		mockUtils.FileWriteErrors = map[string]error{
+			"/tmp/order.toml": fmt.Errorf("unable to write to file"),
+		}
+		testOrder := Order{
+			Order: []OrderEntry{{
+				Group: []BuildpackRef{{
+					ID:       "test",
+					Version:  "0.0.1",
+					Optional: true,
+				}},
+			}},
+			Utils: mockUtils,
+		}
+
+		err := testOrder.Save("/tmp/order.toml")
+
+		assert.Error(t, err, "unable to write to file")
+		assert.False(t, mockUtils.HasWrittenFile("/tmp/order.toml"))
+	})
+}

pkg/cnbutils/utils.go

@@ -0,0 +1,13 @@
+package cnbutils
+
+import (
+	"github.com/SAP/jenkins-library/pkg/command"
+	"github.com/SAP/jenkins-library/pkg/docker"
+	"github.com/SAP/jenkins-library/pkg/piperutils"
+)
+
+type BuildUtils interface {
+	command.ExecRunner
+	piperutils.FileUtils
+	docker.Download
+}

pkg/mock/fileUtils.go

@@ -237,6 +237,11 @@ func (f *FilesMock) FileWrite(path string, content []byte, mode os.FileMode) err
 	return nil
 }
 
+// RemoveAll is a proxy for FileRemove
+func (f *FilesMock) RemoveAll(path string) error {
+	return f.FileRemove(path)
+}
+
 // FileRemove deletes the association of the given path with any content and records the removal of the file.
 // If the path has not been registered before, it returns an error.
 func (f *FilesMock) FileRemove(path string) error {
@@ -310,6 +315,21 @@ func (f *FilesMock) FileRename(oldPath, newPath string) error {
 	return nil
 }
 
+// TempDir create a temp-styled directory in the in-memory, so that this path is established to exist.
+func (f *FilesMock) TempDir(_, pattern string) (string, error) {
+	tmpDir := "/tmp/test"
+
+	if pattern != "" {
+		tmpDir = fmt.Sprintf("/tmp/%stest", pattern)
+	}
+
+	err := f.MkdirAll(tmpDir, 0755)
+	if err != nil {
+		return "", err
+	}
+	return tmpDir, nil
+}
+
 // MkdirAll creates a directory in the in-memory file system, so that this path is established to exist.
 func (f *FilesMock) MkdirAll(path string, mode os.FileMode) error {
 	// NOTE: FilesMock could be extended to have a set of paths for which MkdirAll should fail.

pkg/mock/fileUtils_test.go

@@ -1,11 +1,12 @@
 package mock
 
 import (
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestFilesMockFileExists(t *testing.T) {
@@ -619,3 +620,25 @@ func TestOpen(t *testing.T) {
 		}
 	})
 }
+
+func TestFilesMockTempDir(t *testing.T) {
+	t.Parallel()
+	t.Run("creates a temp dir without a pattern", func(t *testing.T) {
+		files := FilesMock{}
+		dir, err := files.TempDir("", "")
+		assert.NoError(t, err)
+		assert.Equal(t, "/tmp/test", dir)
+		ok, err := files.DirExists("/tmp/test")
+		assert.NoError(t, err)
+		assert.True(t, ok)
+	})
+	t.Run("creates a temp dir with a pattern", func(t *testing.T) {
+		files := FilesMock{}
+		dir, err := files.TempDir("", "pattern")
+		assert.NoError(t, err)
+		assert.Equal(t, "/tmp/patterntest", dir)
+		ok, err := files.DirExists("/tmp/patterntest")
+		assert.NoError(t, err)
+		assert.True(t, ok)
+	})
+}

pkg/piperutils/FileUtils.go

@@ -23,6 +23,10 @@ type FileUtils interface {
 	Chmod(path string, mode os.FileMode) error
 	Glob(pattern string) (matches []string, err error)
 	Chdir(path string) error
+	TempDir(string, string) (string, error)
+	RemoveAll(string) error
+	FileRename(string, string) error
+	Getwd() (string, error)
 }
 
 // Files ...

resources/metadata/cnbBuild.yaml

@@ -48,6 +48,13 @@ spec:
         resourceRef:
           - name: commonPipelineEnvironment
             param: container/registryUrl
+      - name: buildpacks
+        type: "[]string"
+        description: List of custom buildpacks to use in the form of '<hostname>/<repo>[:<tag>]'.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
       - name: path
         type: string
         description: The path should either point to your sources or an artifact build before.

test/groovy/CnbBuildTest.groovy

@@ -43,10 +43,11 @@ public class CnbBuildTest extends BasePiperTest {
             }
         )
 
-        stepRule.step.cnbBuild(script: nullScript, containerImageName: 'foo', containerImageTag: 'bar', containerRegistryUrl: 'test', dockerConfigJsonCredentialsId: 'DOCKER_CREDENTIALS')
+        stepRule.step.cnbBuild(script: nullScript, buildpacks: ['test1', 'test2'], containerImageName: 'foo', containerImageTag: 'bar', containerRegistryUrl: 'test', dockerConfigJsonCredentialsId: 'DOCKER_CREDENTIALS')
 
-        assertThat(calledWithParameters.size(), is(5))
+        assertThat(calledWithParameters.size(), is(6))
         assertThat(calledWithParameters.script, is(nullScript))
+        assertThat(calledWithParameters.buildpacks, is(['test1', 'test2']))
         assertThat(calledWithParameters.containerImageName, is('foo'))
         assertThat(calledWithParameters.containerImageTag, is('bar'))
         assertThat(calledWithParameters.containerRegistryUrl, is('test'))