feat(cnbBuild): cache buildpacks during multi-image build (#3635)

Co-authored-by: Ralf Pannemans <ralf.pannemans@sap.com>
Co-authored-by: Johannes Dillmann <j.dillmann@sap.com>

cmd/containerSaveImage_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/SAP/jenkins-library/pkg/telemetry"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/fake"
 )
 

cmd/malwareExecuteScan_test.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/fake"
 
 	piperDocker "github.com/SAP/jenkins-library/pkg/docker"
@@ -298,3 +298,8 @@ func (c *dockerClientMock) DownloadImage(imageSource, filePath string) (v1.Image
 func (c *dockerClientMock) DownloadImageContent(imageSource, filePath string) (v1.Image, error) {
 	return &fake.FakeImage{}, nil // fmt.Errorf("%s", filePath)
 }
+
+// GetRemoteImageInfo return remote image information
+func (c *dockerClientMock) GetRemoteImageInfo(imageSoure string) (v1.Image, error) {
+	return &fake.FakeImage{}, nil
+}

cmd/protecodeExecuteScan_test.go

@@ -17,11 +17,10 @@ import (
 	piperHttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/SAP/jenkins-library/pkg/mock"
 	"github.com/SAP/jenkins-library/pkg/protecode"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/fake"
 )
 

integration/integration_cnb_test.go

@@ -14,7 +14,10 @@ import (
 	"github.com/testcontainers/testcontainers-go"
 )
 
-var registryURL = "localhost:5000"
+const (
+	registryURL = "localhost:5000"
+	baseBuilder = "paketobuildpacks/builder:0.2.17-base"
+)
 
 func setupDockerRegistry(t *testing.T, ctx context.Context) testcontainers.Container {
 	reqRegistry := testcontainers.ContainerRequest{
@@ -38,14 +41,14 @@ func TestNpmProject(t *testing.T) {
 	defer registryContainer.Terminate(ctx)
 
 	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
-		Image:   "paketobuildpacks/builder:0.1.342-full",
+		Image:   baseBuilder,
 		User:    "cnb",
 		TestDir: []string{"testdata"},
 		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
 	})
 
 	container2 := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
-		Image:   "paketobuildpacks/builder:0.1.342-full",
+		Image:   baseBuilder,
 		User:    "cnb",
 		TestDir: []string{"testdata"},
 		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@@ -79,7 +82,7 @@ func TestProjectDescriptor(t *testing.T) {
 	defer registryContainer.Terminate(ctx)
 
 	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
-		Image:   "paketobuildpacks/builder:0.1.342-full",
+		Image:   baseBuilder,
 		User:    "cnb",
 		TestDir: []string{"testdata", "TestCnbIntegration", "project"},
 		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@@ -109,7 +112,7 @@ func TestZipPath(t *testing.T) {
 	defer registryContainer.Terminate(ctx)
 
 	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
-		Image:   "paketobuildpacks/builder:0.1.342-full",
+		Image:   baseBuilder,
 		User:    "cnb",
 		TestDir: []string{"testdata", "TestCnbIntegration", "zip"},
 		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@@ -133,7 +136,7 @@ func TestNonZipPath(t *testing.T) {
 	defer registryContainer.Terminate(ctx)
 
 	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
-		Image:   "paketobuildpacks/builder:0.1.342-full",
+		Image:   baseBuilder,
 		User:    "cnb",
 		TestDir: []string{"testdata", "TestMtaIntegration", "npm"},
 		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@@ -152,7 +155,7 @@ func TestNpmCustomBuildpacksFullProject(t *testing.T) {
 	defer registryContainer.Terminate(ctx)
 
 	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
-		Image:   "paketobuildpacks/builder:0.1.342-full",
+		Image:   baseBuilder,
 		User:    "cnb",
 		TestDir: []string{"testdata", "TestMtaIntegration", "npm"},
 		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@@ -161,7 +164,7 @@ func TestNpmCustomBuildpacksFullProject(t *testing.T) {
 	container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--buildpacks", "gcr.io/paketo-buildpacks/nodejs:0.14.0", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL)
 
 	container.assertHasOutput(t, "Setting custom buildpacks: '[gcr.io/paketo-buildpacks/nodejs:0.14.0]'")
-	container.assertHasOutput(t, "Downloading buildpack 'gcr.io/paketo-buildpacks/nodejs:0.14.0' to /tmp/nodejs")
+	container.assertHasOutput(t, "Downloading buildpack 'gcr.io/paketo-buildpacks/nodejs:0.14.0' to /tmp/buildpacks_cache/sha256:")
 	container.assertHasOutput(t, "running command: /cnb/lifecycle/creator")
 	container.assertHasOutput(t, "Paketo NPM Start Buildpack")
 	container.assertHasOutput(t, fmt.Sprintf("Saving %s/not-found:0.0.1", registryURL))
@@ -186,7 +189,7 @@ func TestNpmCustomBuildpacksBuildpacklessProject(t *testing.T) {
 	container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--buildpacks", "gcr.io/paketo-buildpacks/nodejs:0.14.0", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL)
 
 	container.assertHasOutput(t, "Setting custom buildpacks: '[gcr.io/paketo-buildpacks/nodejs:0.14.0]'")
-	container.assertHasOutput(t, "Downloading buildpack 'gcr.io/paketo-buildpacks/nodejs:0.14.0' to /tmp/nodejs")
+	container.assertHasOutput(t, "Downloading buildpack 'gcr.io/paketo-buildpacks/nodejs:0.14.0' to /tmp/buildpacks_cache/sha256:")
 	container.assertHasOutput(t, "running command: /cnb/lifecycle/creator")
 	container.assertHasOutput(t, "Paketo NPM Start Buildpack")
 	container.assertHasOutput(t, fmt.Sprintf("Saving %s/not-found:0.0.1", registryURL))
@@ -215,7 +218,7 @@ func TestBindings(t *testing.T) {
 	defer registryContainer.Terminate(ctx)
 
 	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
-		Image:   "paketobuildpacks/builder:0.1.342-full",
+		Image:   baseBuilder,
 		User:    "cnb",
 		TestDir: []string{"testdata"},
 		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@@ -236,7 +239,7 @@ func TestMultiImage(t *testing.T) {
 	defer registryContainer.Terminate(ctx)
 
 	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
-		Image:   "paketobuildpacks/builder:0.1.342-full",
+		Image:   baseBuilder,
 		User:    "cnb",
 		TestDir: []string{"testdata", "TestCnbIntegration"},
 		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@@ -248,6 +251,8 @@ func TestMultiImage(t *testing.T) {
 	container.assertHasOutput(t, "Saving localhost:5000/io-buildpacks-my-app:latest...")
 	container.assertHasOutput(t, "Previous image with name \"localhost:5000/go-app:v1.0.0\" not found")
 	container.assertHasOutput(t, "Saving localhost:5000/go-app:v1.0.0...")
+	container.assertHasOutput(t, "Using cached buildpack")
+	container.assertHasOutput(t, "Saving localhost:5000/my-app2:latest...")
 	container.terminate(t)
 }
 
@@ -258,7 +263,7 @@ func TestPreserveFiles(t *testing.T) {
 	defer registryContainer.Terminate(ctx)
 
 	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
-		Image:   "paketobuildpacks/builder:0.1.342-full",
+		Image:   baseBuilder,
 		User:    "cnb",
 		TestDir: []string{"testdata", "TestCnbIntegration"},
 		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@@ -277,7 +282,7 @@ func TestPreserveFilesIgnored(t *testing.T) {
 	defer registryContainer.Terminate(ctx)
 
 	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
-		Image:   "paketobuildpacks/builder:0.1.342-full",
+		Image:   baseBuilder,
 		User:    "cnb",
 		TestDir: []string{"testdata", "TestCnbIntegration"},
 		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),

integration/testdata/TestCnbIntegration/config_multi_image.yml

@@ -10,3 +10,5 @@ steps:
     - containerImageName: go-app
       containerImageTag: v1.0.0
       path: zip/go.zip
+    - path: project
+      containerImageName: my-app2

pkg/cnbutils/buildpack.go

@@ -3,14 +3,15 @@ package cnbutils
 
 import (
 	"encoding/json"
-	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/pkg/errors"
 )
 
+const bpCacheDir = "/tmp/buildpacks_cache"
+
 type BuildPackMetadata struct {
 	ID          string    `toml:"id,omitempty" json:"id,omitempty" yaml:"id,omitempty"`
 	Name        string    `toml:"name,omitempty" json:"name,omitempty" yaml:"name,omitempty"`
@@ -37,33 +38,57 @@ func DownloadBuildpacks(path string, bpacks []string, dockerCreds string, utils
 		Utils: utils,
 	}
 
+	err := utils.MkdirAll(bpCacheDir, os.ModePerm)
+	if err != nil {
+		return Order{}, errors.Wrap(err, "failed to create temp directory for buildpack cache")
+	}
+
 	for _, bpack := range bpacks {
 		var bpackMeta BuildPackMetadata
-		tempDir, err := utils.TempDir("", filepath.Base(bpack))
+		imageInfo, err := utils.GetRemoteImageInfo(bpack)
 		if err != nil {
-			return Order{}, fmt.Errorf("failed to create temp directory, error: %s", err.Error())
+			return Order{}, errors.Wrap(err, "failed to get remote image info of buildpack")
 		}
-		defer utils.RemoveAll(tempDir)
+		hash, err := imageInfo.Digest()
-
-		log.Entry().Infof("Downloading buildpack '%s' to %s", bpack, tempDir)
-		img, err := utils.DownloadImageContent(bpack, tempDir)
 		if err != nil {
-			return Order{}, fmt.Errorf("failed download buildpack image '%s', error: %s", bpack, err.Error())
+			return Order{}, errors.Wrap(err, "failed to get image digest")
+		}
+		cacheDir := filepath.Join(bpCacheDir, hash.String())
+
+		cacheExists, err := utils.DirExists(cacheDir)
+		if err != nil {
+			return Order{}, errors.Wrapf(err, "failed to check if cache dir '%s' exists", cacheDir)
 		}
 
-		imgConf, err := img.ConfigFile()
+		if cacheExists {
+			log.Entry().Infof("Using cached buildpack '%s'", bpack)
+		} else {
+			err := utils.MkdirAll(cacheDir, os.ModePerm)
+			if err != nil {
+				return Order{}, errors.Wrap(err, "failed to create temp directory for buildpack cache")
+			}
+
+			log.Entry().Infof("Downloading buildpack '%s' to %s", bpack, cacheDir)
+			img, err := utils.DownloadImageContent(bpack, cacheDir)
+			if err != nil {
+				return Order{}, errors.Wrapf(err, "failed download buildpack image '%s'", bpack)
+			}
+			imageInfo = img
+		}
+
+		imgConf, err := imageInfo.ConfigFile()
 		if err != nil {
-			return Order{}, fmt.Errorf("failed to read '%s' image config, error: %s", bpack, err.Error())
+			return Order{}, errors.Wrapf(err, "failed to read '%s' image config", bpack)
 		}
 
 		err = json.Unmarshal([]byte(imgConf.Config.Labels["io.buildpacks.buildpackage.metadata"]), &bpackMeta)
 		if err != nil {
-			return Order{}, fmt.Errorf("failed unmarshal '%s' image label, error: %s", bpack, err.Error())
+			return Order{}, errors.Wrapf(err, "failed unmarshal '%s' image label", bpack)
 		}
 		log.Entry().Debugf("Buildpack metadata: '%v'", bpackMeta)
 		orderEntry.Group = append(orderEntry.Group, bpackMeta)
 
-		err = copyBuildPack(filepath.Join(tempDir, "cnb/buildpacks"), path, utils)
+		err = CopyProject(filepath.Join(cacheDir, "cnb/buildpacks"), path, nil, nil, utils)
 		if err != nil {
 			return Order{}, err
 		}
@@ -73,38 +98,3 @@ func DownloadBuildpacks(path string, bpacks []string, dockerCreds string, utils
 
 	return order, nil
 }
-
-func copyBuildPack(src, dst string, utils BuildUtils) error {
-	buildpacks, err := utils.Glob(filepath.Join(src, "*"))
-	if err != nil {
-		return fmt.Errorf("failed to read directory: %s, error: %s", src, err.Error())
-	}
-
-	for _, buildpack := range buildpacks {
-		versions, err := utils.Glob(filepath.Join(buildpack, "*"))
-		if err != nil {
-			return fmt.Errorf("failed to read directory: %s, error: %s", buildpack, err.Error())
-		}
-		for _, srcVersionPath := range versions {
-			destVersionPath := filepath.Join(dst, strings.ReplaceAll(srcVersionPath, src, ""))
-
-			exists, err := utils.FileExists(destVersionPath)
-			if err != nil {
-				return fmt.Errorf("failed to check if directory exists: '%s', error: '%s'", destVersionPath, err.Error())
-			}
-			if exists {
-				utils.RemoveAll(destVersionPath)
-			}
-
-			if err := utils.MkdirAll(filepath.Dir(destVersionPath), 0755); err != nil {
-				return fmt.Errorf("failed to create directory: '%s', error: '%s'", filepath.Dir(destVersionPath), err.Error())
-			}
-
-			err = utils.FileRename(srcVersionPath, destVersionPath)
-			if err != nil {
-				return fmt.Errorf("failed to move '%s' to '%s', error: %s", srcVersionPath, destVersionPath, err.Error())
-			}
-		}
-	}
-	return nil
-}

pkg/cnbutils/buildpack_test.go

@@ -14,11 +14,9 @@ func TestBuildpackDownload(t *testing.T) {
 		FilesMock:      &mock.FilesMock{},
 	}
 
-	t.Run("successfully downloads a buildpack", func(t *testing.T) {
+	t.Run("it creates an order object", func(t *testing.T) {
-		mockUtils.AddDir("/tmp/testtest")
+		order, err := cnbutils.DownloadBuildpacks("/destination", []string{"buildpack"}, "/tmp/config.json", mockUtils)
-		_, err := cnbutils.DownloadBuildpacks("/test", []string{"test"}, "/test/config.json", mockUtils)
-
 		assert.NoError(t, err)
-		assert.True(t, mockUtils.HasRemovedFile("/tmp/testtest"))
+		assert.Equal(t, 1, len(order.Order))
 	})
 }

pkg/cnbutils/mock.go

@@ -44,3 +44,17 @@ func (c *MockUtils) DownloadImage(src, dst string) (v1.Image, error) {
 func (c *MockUtils) GetImageSource() (string, error) {
 	return "imageSource", nil
 }
+
+func (c *MockUtils) GetRemoteImageInfo(imageSource string) (v1.Image, error) {
+	fakeImage := fakeImage.FakeImage{}
+	fakeImage.ConfigFileReturns(&v1.ConfigFile{
+		Config: v1.Config{
+			Labels: map[string]string{
+				"io.buildpacks.buildpackage.metadata": "{\"id\": \"testbuildpack\", \"version\": \"0.0.1\"}",
+			},
+		},
+	}, nil)
+	fakeImage.DigestReturns(v1.Hash{}, nil)
+
+	return &fakeImage, nil
+}

pkg/cnbutils/report.go

@@ -3,7 +3,6 @@ package cnbutils
 import (
 	"fmt"
 
-	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/buildpacks/lifecycle/platform"
 	"github.com/pelletier/go-toml"
 )
@@ -23,8 +22,6 @@ func DigestFromReport(utils BuildUtils) (string, error) {
 		return "", err
 	}
 
-	log.Entry().Debugf("Image report: %#v\n", report)
-
 	if report.Image.Digest == "" {
 		return "", fmt.Errorf("image digest is empty")
 	}

pkg/docker/docker.go

@@ -11,11 +11,14 @@ import (
 
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
+	"github.com/pkg/errors"
 
 	cranecmd "github.com/google/go-containerregistry/cmd/crane/cmd"
+	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/google/go-containerregistry/pkg/name"
-	"github.com/google/go-containerregistry/pkg/v1"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 )
 
 // AuthEntry defines base64 encoded username:password required inside a Docker config.json
@@ -94,6 +97,7 @@ type ClientOptions struct {
 type Download interface {
 	DownloadImage(imageSource, targetFile string) (v1.Image, error)
 	DownloadImageContent(imageSource, targetDir string) (v1.Image, error)
+	GetRemoteImageInfo(string) (v1.Image, error)
 }
 
 // SetOptions sets options used for the docker client
@@ -103,7 +107,7 @@ func (c *Client) SetOptions(options ClientOptions) {
 	c.localPath = options.LocalPath
 }
 
-//DownloadImageToPath downloads the image content into the given targetDir. Returns with an error if the targetDir doesnt exist
+//DownloadImageContent downloads the image content into the given targetDir. Returns with an error if the targetDir doesnt exist
 func (c *Client) DownloadImageContent(imageSource, targetDir string) (v1.Image, error) {
 	if fileInfo, err := os.Stat(targetDir); err != nil {
 		return nil, err
@@ -178,6 +182,16 @@ func (c *Client) DownloadImage(imageSource, targetFile string) (v1.Image, error)
 	return img, nil
 }
 
+// GetRemoteImageInfo retrieves information about the image (e.g. digest) without actually downoading it
+func (c *Client) GetRemoteImageInfo(imageSource string) (v1.Image, error) {
+	ref, err := c.getImageRef(imageSource)
+	if err != nil {
+		return nil, errors.Wrap(err, "parsing image reference")
+	}
+
+	return remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
+}
+
 func (c *Client) getImageRef(image string) (name.Reference, error) {
 	opts := []name.Option{}
 

pkg/mock/dockerClient.go

@@ -2,17 +2,20 @@ package mock
 
 import (
 	"fmt"
-	"github.com/google/go-containerregistry/pkg/v1"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 )
 
 // DownloadMock .
 type DownloadMock struct {
-	FilePath    string
+	FilePath       string
-	ImageRef    string
+	ImageRef       string
-	RegistryURL string
+	RemoteImageRef string
+	RegistryURL    string
 
-	ReturnImage v1.Image
+	ReturnImage     v1.Image
-	ReturnError string
+	RemoteImageInfo v1.Image
+	ReturnError     string
 
 	Stub func(imageRef, targetDir string) (v1.Image, error)
 }
@@ -42,3 +45,14 @@ func (c *DownloadMock) DownloadImageContent(imageRef, targetFile string) (v1.Ima
 	}
 	return c.ReturnImage, nil
 }
+
+// GetRemoteImageInfo .
+func (c *DownloadMock) GetRemoteImageInfo(imageRef string) (v1.Image, error) {
+	c.RemoteImageRef = imageRef
+
+	if len(c.ReturnError) > 0 {
+		return nil, fmt.Errorf(c.ReturnError)
+	}
+
+	return c.RemoteImageInfo, nil
+}

pkg/piperutils/FileUtils.go

@@ -115,6 +115,12 @@ func (f Files) Copy(src, dst string) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
+	stats, err := os.Stat(src)
+	if err != nil {
+		return 0, err
+	}
+
+	os.Chmod(dst, stats.Mode())
 	defer func() { _ = destination.Close() }()
 	nBytes, err := CopyData(destination, source)
 	return nBytes, err
@@ -236,7 +242,7 @@ func Untar(src string, dest string, stripComponentLevel int) error {
 	defer file.Close()
 
 	if err != nil {
-		fmt.Errorf("unable to open src: %v", err)
+		return fmt.Errorf("unable to open src: %v", err)
 	}
 
 	if b, err := isFileGzipped(src); err == nil && b {