jenkins/sap-jenkins-library
1
0
Fork 0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-01-30 05:59:39 +02:00
Code Issues Releases Activity

feat(cnbBuild): cache buildpacks during multi-image build (#3635)

Browse Source 
Co-authored-by: Ralf Pannemans <ralf.pannemans@sap.com>
Co-authored-by: Johannes Dillmann <j.dillmann@sap.com>
This commit is contained in:
Pavel Busko 2022-03-30 13:58:16 +02:00 committed by GitHub
parent f4f11dba7f
commit 1f750af16d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 126 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/containerSaveImage_test.go
View File

@ -11,7 +11,7 @@ import (
"github.com/SAP/jenkins-library/pkg/telemetry"
"github.com/stretchr/testify/assert"
"github.com/google/go-containerregistry/pkg/v1"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/fake"
)

7
cmd/malwareExecuteScan_test.go
View File

@ -9,7 +9,7 @@ import (
"strings"
"testing"
"github.com/google/go-containerregistry/pkg/v1"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/fake"
piperDocker "github.com/SAP/jenkins-library/pkg/docker"
@ -298,3 +298,8 @@ func (c *dockerClientMock) DownloadImage(imageSource, filePath string) (v1.Image
func (c *dockerClientMock) DownloadImageContent(imageSource, filePath string) (v1.Image, error) {
return &fake.FakeImage{}, nil // fmt.Errorf("%s", filePath)
}
// GetRemoteImageInfo return remote image information
func (c *dockerClientMock) GetRemoteImageInfo(imageSoure string) (v1.Image, error) {
return &fake.FakeImage{}, nil
}

3
cmd/protecodeExecuteScan_test.go
View File

@ -17,11 +17,10 @@ import (
piperHttp "github.com/SAP/jenkins-library/pkg/http"
"github.com/SAP/jenkins-library/pkg/mock"
"github.com/SAP/jenkins-library/pkg/protecode"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/google/go-containerregistry/pkg/v1"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/fake"
)

31
integration/integration_cnb_test.go
View File

@ -14,7 +14,10 @@ import (
"github.com/testcontainers/testcontainers-go"
)
var registryURL = "localhost:5000"
const (
registryURL = "localhost:5000"
baseBuilder = "paketobuildpacks/builder:0.2.17-base"
)
func setupDockerRegistry(t *testing.T, ctx context.Context) testcontainers.Container {
reqRegistry := testcontainers.ContainerRequest{
@ -38,14 +41,14 @@ func TestNpmProject(t *testing.T) {
defer registryContainer.Terminate(ctx)
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "paketobuildpacks/builder:0.1.342-full",
Image: baseBuilder,
User: "cnb",
TestDir: []string{"testdata"},
Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
})
container2 := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "paketobuildpacks/builder:0.1.342-full",
Image: baseBuilder,
User: "cnb",
TestDir: []string{"testdata"},
Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@ -79,7 +82,7 @@ func TestProjectDescriptor(t *testing.T) {
defer registryContainer.Terminate(ctx)
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "paketobuildpacks/builder:0.1.342-full",
Image: baseBuilder,
User: "cnb",
TestDir: []string{"testdata", "TestCnbIntegration", "project"},
Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@ -109,7 +112,7 @@ func TestZipPath(t *testing.T) {
defer registryContainer.Terminate(ctx)
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "paketobuildpacks/builder:0.1.342-full",
Image: baseBuilder,
User: "cnb",
TestDir: []string{"testdata", "TestCnbIntegration", "zip"},
Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@ -133,7 +136,7 @@ func TestNonZipPath(t *testing.T) {
defer registryContainer.Terminate(ctx)
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "paketobuildpacks/builder:0.1.342-full",
Image: baseBuilder,
User: "cnb",
TestDir: []string{"testdata", "TestMtaIntegration", "npm"},
Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@ -152,7 +155,7 @@ func TestNpmCustomBuildpacksFullProject(t *testing.T) {
defer registryContainer.Terminate(ctx)
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "paketobuildpacks/builder:0.1.342-full",
Image: baseBuilder,
User: "cnb",
TestDir: []string{"testdata", "TestMtaIntegration", "npm"},
Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@ -161,7 +164,7 @@ func TestNpmCustomBuildpacksFullProject(t *testing.T) {
container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--buildpacks", "gcr.io/paketo-buildpacks/nodejs:0.14.0", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL)
container.assertHasOutput(t, "Setting custom buildpacks: '[gcr.io/paketo-buildpacks/nodejs:0.14.0]'")
container.assertHasOutput(t, "Downloading buildpack 'gcr.io/paketo-buildpacks/nodejs:0.14.0' to /tmp/nodejs")
container.assertHasOutput(t, "Downloading buildpack 'gcr.io/paketo-buildpacks/nodejs:0.14.0' to /tmp/buildpacks_cache/sha256:")
container.assertHasOutput(t, "running command: /cnb/lifecycle/creator")
container.assertHasOutput(t, "Paketo NPM Start Buildpack")
container.assertHasOutput(t, fmt.Sprintf("Saving %s/not-found:0.0.1", registryURL))
@ -186,7 +189,7 @@ func TestNpmCustomBuildpacksBuildpacklessProject(t *testing.T) {
container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--buildpacks", "gcr.io/paketo-buildpacks/nodejs:0.14.0", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL)
container.assertHasOutput(t, "Setting custom buildpacks: '[gcr.io/paketo-buildpacks/nodejs:0.14.0]'")
container.assertHasOutput(t, "Downloading buildpack 'gcr.io/paketo-buildpacks/nodejs:0.14.0' to /tmp/nodejs")
container.assertHasOutput(t, "Downloading buildpack 'gcr.io/paketo-buildpacks/nodejs:0.14.0' to /tmp/buildpacks_cache/sha256:")
container.assertHasOutput(t, "running command: /cnb/lifecycle/creator")
container.assertHasOutput(t, "Paketo NPM Start Buildpack")
container.assertHasOutput(t, fmt.Sprintf("Saving %s/not-found:0.0.1", registryURL))
@ -215,7 +218,7 @@ func TestBindings(t *testing.T) {
defer registryContainer.Terminate(ctx)
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "paketobuildpacks/builder:0.1.342-full",
Image: baseBuilder,
User: "cnb",
TestDir: []string{"testdata"},
Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@ -236,7 +239,7 @@ func TestMultiImage(t *testing.T) {
defer registryContainer.Terminate(ctx)
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "paketobuildpacks/builder:0.1.342-full",
Image: baseBuilder,
User: "cnb",
TestDir: []string{"testdata", "TestCnbIntegration"},
Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@ -248,6 +251,8 @@ func TestMultiImage(t *testing.T) {
container.assertHasOutput(t, "Saving localhost:5000/io-buildpacks-my-app:latest...")
container.assertHasOutput(t, "Previous image with name \"localhost:5000/go-app:v1.0.0\" not found")
container.assertHasOutput(t, "Saving localhost:5000/go-app:v1.0.0...")
container.assertHasOutput(t, "Using cached buildpack")
container.assertHasOutput(t, "Saving localhost:5000/my-app2:latest...")
container.terminate(t)
}
@ -258,7 +263,7 @@ func TestPreserveFiles(t *testing.T) {
defer registryContainer.Terminate(ctx)
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "paketobuildpacks/builder:0.1.342-full",
Image: baseBuilder,
User: "cnb",
TestDir: []string{"testdata", "TestCnbIntegration"},
Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
@ -277,7 +282,7 @@ func TestPreserveFilesIgnored(t *testing.T) {
defer registryContainer.Terminate(ctx)
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "paketobuildpacks/builder:0.1.342-full",
Image: baseBuilder,
User: "cnb",
TestDir: []string{"testdata", "TestCnbIntegration"},
Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),

2
integration/testdata/TestCnbIntegration/config_multi_image.yml vendored
View File

@ -10,3 +10,5 @@ steps:
- containerImageName: go-app
containerImageTag: v1.0.0
path: zip/go.zip
- path: project
containerImageName: my-app2

86
pkg/cnbutils/buildpack.go
View File

@ -3,14 +3,15 @@ package cnbutils
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/pkg/errors"
)
const bpCacheDir = "/tmp/buildpacks_cache"
type BuildPackMetadata struct {
ID string `toml:"id,omitempty" json:"id,omitempty" yaml:"id,omitempty"`
Name string `toml:"name,omitempty" json:"name,omitempty" yaml:"name,omitempty"`
@ -37,33 +38,57 @@ func DownloadBuildpacks(path string, bpacks []string, dockerCreds string, utils
Utils: utils,
}
err := utils.MkdirAll(bpCacheDir, os.ModePerm)
if err != nil {
return Order{}, errors.Wrap(err, "failed to create temp directory for buildpack cache")
}
for _, bpack := range bpacks {
var bpackMeta BuildPackMetadata
tempDir, err := utils.TempDir("", filepath.Base(bpack))
imageInfo, err := utils.GetRemoteImageInfo(bpack)
if err != nil {
return Order{}, fmt.Errorf("failed to create temp directory, error: %s", err.Error())
return Order{}, errors.Wrap(err, "failed to get remote image info of buildpack")
}
defer utils.RemoveAll(tempDir)
log.Entry().Infof("Downloading buildpack '%s' to %s", bpack, tempDir)
img, err := utils.DownloadImageContent(bpack, tempDir)
hash, err := imageInfo.Digest()
if err != nil {
return Order{}, fmt.Errorf("failed download buildpack image '%s', error: %s", bpack, err.Error())
return Order{}, errors.Wrap(err, "failed to get image digest")
}
cacheDir := filepath.Join(bpCacheDir, hash.String())
cacheExists, err := utils.DirExists(cacheDir)
if err != nil {
return Order{}, errors.Wrapf(err, "failed to check if cache dir '%s' exists", cacheDir)
}
imgConf, err := img.ConfigFile()
if cacheExists {
log.Entry().Infof("Using cached buildpack '%s'", bpack)
} else {
err := utils.MkdirAll(cacheDir, os.ModePerm)
if err != nil {
return Order{}, errors.Wrap(err, "failed to create temp directory for buildpack cache")
}
log.Entry().Infof("Downloading buildpack '%s' to %s", bpack, cacheDir)
img, err := utils.DownloadImageContent(bpack, cacheDir)
if err != nil {
return Order{}, errors.Wrapf(err, "failed download buildpack image '%s'", bpack)
}
imageInfo = img
}
imgConf, err := imageInfo.ConfigFile()
if err != nil {
return Order{}, fmt.Errorf("failed to read '%s' image config, error: %s", bpack, err.Error())
return Order{}, errors.Wrapf(err, "failed to read '%s' image config", bpack)
}
err = json.Unmarshal([]byte(imgConf.Config.Labels["io.buildpacks.buildpackage.metadata"]), &bpackMeta)
if err != nil {
return Order{}, fmt.Errorf("failed unmarshal '%s' image label, error: %s", bpack, err.Error())
return Order{}, errors.Wrapf(err, "failed unmarshal '%s' image label", bpack)
}
log.Entry().Debugf("Buildpack metadata: '%v'", bpackMeta)
orderEntry.Group = append(orderEntry.Group, bpackMeta)
err = copyBuildPack(filepath.Join(tempDir, "cnb/buildpacks"), path, utils)
err = CopyProject(filepath.Join(cacheDir, "cnb/buildpacks"), path, nil, nil, utils)
if err != nil {
return Order{}, err
}
@ -73,38 +98,3 @@ func DownloadBuildpacks(path string, bpacks []string, dockerCreds string, utils
return order, nil
}
func copyBuildPack(src, dst string, utils BuildUtils) error {
buildpacks, err := utils.Glob(filepath.Join(src, "*"))
if err != nil {
return fmt.Errorf("failed to read directory: %s, error: %s", src, err.Error())
}
for _, buildpack := range buildpacks {
versions, err := utils.Glob(filepath.Join(buildpack, "*"))
if err != nil {
return fmt.Errorf("failed to read directory: %s, error: %s", buildpack, err.Error())
}
for _, srcVersionPath := range versions {
destVersionPath := filepath.Join(dst, strings.ReplaceAll(srcVersionPath, src, ""))
exists, err := utils.FileExists(destVersionPath)
if err != nil {
return fmt.Errorf("failed to check if directory exists: '%s', error: '%s'", destVersionPath, err.Error())
}
if exists {
utils.RemoveAll(destVersionPath)
}
if err := utils.MkdirAll(filepath.Dir(destVersionPath), 0755); err != nil {
return fmt.Errorf("failed to create directory: '%s', error: '%s'", filepath.Dir(destVersionPath), err.Error())
}
err = utils.FileRename(srcVersionPath, destVersionPath)
if err != nil {
return fmt.Errorf("failed to move '%s' to '%s', error: %s", srcVersionPath, destVersionPath, err.Error())
}
}
}
return nil
}

8
pkg/cnbutils/buildpack_test.go
View File

@ -14,11 +14,9 @@ func TestBuildpackDownload(t *testing.T) {
FilesMock: &mock.FilesMock{},
}
t.Run("successfully downloads a buildpack", func(t *testing.T) {
mockUtils.AddDir("/tmp/testtest")
_, err := cnbutils.DownloadBuildpacks("/test", []string{"test"}, "/test/config.json", mockUtils)
t.Run("it creates an order object", func(t *testing.T) {
order, err := cnbutils.DownloadBuildpacks("/destination", []string{"buildpack"}, "/tmp/config.json", mockUtils)
assert.NoError(t, err)
assert.True(t, mockUtils.HasRemovedFile("/tmp/testtest"))
assert.Equal(t, 1, len(order.Order))
})
}

14
pkg/cnbutils/mock.go
View File

@ -44,3 +44,17 @@ func (c *MockUtils) DownloadImage(src, dst string) (v1.Image, error) {
func (c *MockUtils) GetImageSource() (string, error) {
return "imageSource", nil
}
func (c *MockUtils) GetRemoteImageInfo(imageSource string) (v1.Image, error) {
fakeImage := fakeImage.FakeImage{}
fakeImage.ConfigFileReturns(&v1.ConfigFile{
Config: v1.Config{
Labels: map[string]string{
"io.buildpacks.buildpackage.metadata": "{\"id\": \"testbuildpack\", \"version\": \"0.0.1\"}",
},
},
}, nil)
fakeImage.DigestReturns(v1.Hash{}, nil)
return &fakeImage, nil
}

3
pkg/cnbutils/report.go
View File

@ -3,7 +3,6 @@ package cnbutils
import (
"fmt"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/buildpacks/lifecycle/platform"
"github.com/pelletier/go-toml"
)
@ -23,8 +22,6 @@ func DigestFromReport(utils BuildUtils) (string, error) {
return "", err
}
log.Entry().Debugf("Image report: %#v\n", report)
if report.Image.Digest == "" {
return "", fmt.Errorf("image digest is empty")
}

18
pkg/docker/docker.go
View File

@ -11,11 +11,14 @@ import (
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/pkg/errors"
cranecmd "github.com/google/go-containerregistry/cmd/crane/cmd"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/crane"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
)
// AuthEntry defines base64 encoded username:password required inside a Docker config.json
@ -94,6 +97,7 @@ type ClientOptions struct {
type Download interface {
DownloadImage(imageSource, targetFile string) (v1.Image, error)
DownloadImageContent(imageSource, targetDir string) (v1.Image, error)
GetRemoteImageInfo(string) (v1.Image, error)
}
// SetOptions sets options used for the docker client
@ -103,7 +107,7 @@ func (c *Client) SetOptions(options ClientOptions) {
c.localPath = options.LocalPath
}
//DownloadImageToPath downloads the image content into the given targetDir. Returns with an error if the targetDir doesnt exist
//DownloadImageContent downloads the image content into the given targetDir. Returns with an error if the targetDir doesnt exist
func (c *Client) DownloadImageContent(imageSource, targetDir string) (v1.Image, error) {
if fileInfo, err := os.Stat(targetDir); err != nil {
return nil, err
@ -178,6 +182,16 @@ func (c *Client) DownloadImage(imageSource, targetFile string) (v1.Image, error)
return img, nil
}
// GetRemoteImageInfo retrieves information about the image (e.g. digest) without actually downoading it
func (c *Client) GetRemoteImageInfo(imageSource string) (v1.Image, error) {
ref, err := c.getImageRef(imageSource)
if err != nil {
return nil, errors.Wrap(err, "parsing image reference")
}
return remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
}
func (c *Client) getImageRef(image string) (name.Reference, error) {
opts := []name.Option{}

26
pkg/mock/dockerClient.go
View File

@ -2,17 +2,20 @@ package mock
import (
"fmt"
"github.com/google/go-containerregistry/pkg/v1"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
// DownloadMock .
type DownloadMock struct {
FilePath string
ImageRef string
RegistryURL string
FilePath string
ImageRef string
RemoteImageRef string
RegistryURL string
ReturnImage v1.Image
ReturnError string
ReturnImage v1.Image
RemoteImageInfo v1.Image
ReturnError string
Stub func(imageRef, targetDir string) (v1.Image, error)
}
@ -42,3 +45,14 @@ func (c *DownloadMock) DownloadImageContent(imageRef, targetFile string) (v1.Ima
}
return c.ReturnImage, nil
}
// GetRemoteImageInfo .
func (c *DownloadMock) GetRemoteImageInfo(imageRef string) (v1.Image, error) {
c.RemoteImageRef = imageRef
if len(c.ReturnError) > 0 {
return nil, fmt.Errorf(c.ReturnError)
}
return c.RemoteImageInfo, nil
}

8
pkg/piperutils/FileUtils.go
View File

@ -115,6 +115,12 @@ func (f Files) Copy(src, dst string) (int64, error) {
if err != nil {
return 0, err
}
stats, err := os.Stat(src)
if err != nil {
return 0, err
}
os.Chmod(dst, stats.Mode())
defer func() { _ = destination.Close() }()
nBytes, err := CopyData(destination, source)
return nBytes, err
@ -236,7 +242,7 @@ func Untar(src string, dest string, stripComponentLevel int) error {
defer file.Close()
if err != nil {
fmt.Errorf("unable to open src: %v", err)
return fmt.Errorf("unable to open src: %v", err)
}
if b, err := isFileGzipped(src); err == nil && b {