1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2024-12-14 11:03:09 +02:00
sap-jenkins-library/cmd/cnbBuild.go
Pavel Busko d8dacda121
feat(cnbBuild): support setting registry username and password via parameters (#4426)
* feat(cnbBuild): support setting registry username and password via parameters

* fix gitops integration test assertion

Co-authored-by: Pavel Busko <pavel.busko@sap.com>

* Update integration/integration_gitops_test.go

---------

Co-authored-by: Ralf Pannemans <ralf.pannemans@sap.com>
2023-06-30 12:02:35 +00:00

671 lines
21 KiB
Go

package cmd
import (
"archive/zip"
"encoding/json"
"fmt"
"os"
"path"
"path/filepath"
"github.com/SAP/jenkins-library/pkg/buildsettings"
"github.com/SAP/jenkins-library/pkg/certutils"
"github.com/SAP/jenkins-library/pkg/cnbutils"
"github.com/SAP/jenkins-library/pkg/cnbutils/bindings"
"github.com/SAP/jenkins-library/pkg/cnbutils/privacy"
"github.com/SAP/jenkins-library/pkg/cnbutils/project"
"github.com/SAP/jenkins-library/pkg/cnbutils/project/metadata"
"github.com/SAP/jenkins-library/pkg/command"
"github.com/SAP/jenkins-library/pkg/docker"
piperhttp "github.com/SAP/jenkins-library/pkg/http"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/SAP/jenkins-library/pkg/syft"
"github.com/SAP/jenkins-library/pkg/telemetry"
"github.com/imdario/mergo"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
ignore "github.com/sabhiram/go-gitignore"
)
const (
creatorPath = "/cnb/lifecycle/creator"
platformPath = "/tmp/platform"
)
type pathEnum string
const (
pathEnumRoot = pathEnum("root")
pathEnumFolder = pathEnum("folder")
pathEnumArchive = pathEnum("archive")
)
type cnbBuildUtilsBundle struct {
*command.Command
*piperutils.Files
*docker.Client
}
type cnbBuildTelemetry struct {
dockerImage string
Version int `json:"version"`
Data []cnbBuildTelemetryData `json:"data"`
}
type cnbBuildTelemetryData struct {
ImageTag string `json:"imageTag"`
AdditionalTags []string `json:"additionalTags"`
BindingKeys []string `json:"bindingKeys"`
Path pathEnum `json:"path"`
BuildEnv cnbBuildTelemetryDataBuildEnv `json:"buildEnv"`
Buildpacks cnbBuildTelemetryDataBuildpacks `json:"buildpacks"`
ProjectDescriptor cnbBuildTelemetryDataProjectDescriptor `json:"projectDescriptor"`
BuildTool string `json:"buildTool"`
Builder string `json:"builder"`
}
type cnbBuildTelemetryDataBuildEnv struct {
KeysFromConfig []string `json:"keysFromConfig"`
KeysFromProjectDescriptor []string `json:"keysFromProjectDescriptor"`
KeysOverall []string `json:"keysOverall"`
JVMVersion string `json:"jvmVersion"`
KeyValues map[string]interface{} `json:"keyValues"`
}
type cnbBuildTelemetryDataBuildpacks struct {
FromConfig []string `json:"FromConfig"`
FromProjectDescriptor []string `json:"FromProjectDescriptor"`
Overall []string `json:"overall"`
}
type cnbBuildTelemetryDataProjectDescriptor struct {
Used bool `json:"used"`
IncludeUsed bool `json:"includeUsed"`
ExcludeUsed bool `json:"excludeUsed"`
}
func processConfigs(main cnbBuildOptions, multipleImages []map[string]interface{}) ([]cnbBuildOptions, error) {
var result []cnbBuildOptions
if len(multipleImages) == 0 {
result = append(result, main)
return result, nil
}
for _, conf := range multipleImages {
var structuredConf cnbBuildOptions
err := mapstructure.Decode(conf, &structuredConf)
if err != nil {
return nil, err
}
err = mergo.Merge(&structuredConf, main)
if err != nil {
return nil, err
}
result = append(result, structuredConf)
}
return result, nil
}
func setCustomBuildpacks(bpacks []string, dockerCreds string, utils cnbutils.BuildUtils) (string, string, error) {
buildpacksPath := "/tmp/buildpacks"
orderPath := "/tmp/buildpacks/order.toml"
newOrder, err := cnbutils.DownloadBuildpacks(buildpacksPath, bpacks, dockerCreds, utils)
if err != nil {
return "", "", err
}
err = newOrder.Save(orderPath)
if err != nil {
return "", "", err
}
return buildpacksPath, orderPath, nil
}
func newCnbBuildUtils() cnbutils.BuildUtils {
utils := cnbBuildUtilsBundle{
Command: &command.Command{
StepName: "cnbBuild",
},
Files: &piperutils.Files{},
Client: &docker.Client{},
}
utils.Stdout(log.Writer())
utils.Stderr(log.Writer())
return &utils
}
func cnbBuild(config cnbBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment) {
utils := newCnbBuildUtils()
client := &piperhttp.Client{}
err := callCnbBuild(&config, telemetryData, utils, commonPipelineEnvironment, client)
if err != nil {
log.Entry().WithError(err).Fatal("step execution failed")
}
}
func isBuilder(utils cnbutils.BuildUtils) error {
exists, err := utils.FileExists(creatorPath)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("binary '%s' not found", creatorPath)
}
return nil
}
func isZip(path string) bool {
r, err := zip.OpenReader(path)
switch {
case err == nil:
_ = r.Close()
return true
case err == zip.ErrFormat:
return false
default:
return false
}
}
func cleanDir(dir string, utils cnbutils.BuildUtils) error {
dirContent, err := utils.Glob(filepath.Join(dir, "*"))
if err != nil {
return err
}
for _, obj := range dirContent {
err = utils.RemoveAll(obj)
if err != nil {
return err
}
}
return nil
}
func extractZip(source, target string) error {
if isZip(source) {
log.Entry().Infof("Extracting archive '%s' to '%s'", source, target)
_, err := piperutils.Unzip(source, target)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "Extracting archive '%s' to '%s' failed", source, target)
}
} else {
log.SetErrorCategory(log.ErrorBuild)
return errors.New("application path must be a directory or zip")
}
return nil
}
func renameDockerConfig(config *cnbBuildOptions, utils cnbutils.BuildUtils) error {
if filepath.Base(config.DockerConfigJSON) != "config.json" {
log.Entry().Debugf("Renaming docker config file from '%s' to 'config.json'", filepath.Base(config.DockerConfigJSON))
newPath := filepath.Join(filepath.Dir(config.DockerConfigJSON), "config.json")
alreadyExists, err := utils.FileExists(newPath)
if err != nil {
return err
}
if alreadyExists {
return nil
}
err = utils.FileRename(config.DockerConfigJSON, newPath)
if err != nil {
return err
}
config.DockerConfigJSON = newPath
}
return nil
}
func linkTargetFolder(utils cnbutils.BuildUtils, source, target string) error {
var err error
linkPath := filepath.Join(target, "target")
targetPath := filepath.Join(source, "target")
if ok, _ := utils.DirExists(targetPath); !ok {
err = utils.MkdirAll(targetPath, os.ModePerm)
if err != nil {
return err
}
}
if ok, _ := utils.DirExists(linkPath); ok {
err = utils.RemoveAll(linkPath)
if err != nil {
return err
}
}
return utils.Symlink(targetPath, linkPath)
}
func (config *cnbBuildOptions) mergeEnvVars(vars map[string]interface{}) {
if config.BuildEnvVars == nil {
config.BuildEnvVars = vars
return
}
for k, v := range vars {
_, exists := config.BuildEnvVars[k]
if !exists {
config.BuildEnvVars[k] = v
}
}
}
func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (pathEnum, string, error) {
pwd, err := utils.Getwd()
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return "", "", errors.Wrap(err, "failed to get current working directory")
}
if config.Path == "" {
return pathEnumRoot, pwd, nil
}
matches, err := utils.Glob(config.Path)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return "", "", errors.Wrapf(err, "Failed to resolve glob for '%s'", config.Path)
}
numMatches := len(matches)
if numMatches != 1 {
log.SetErrorCategory(log.ErrorConfiguration)
return "", "", errors.Errorf("Failed to resolve glob for '%s', matching %d file(s)", config.Path, numMatches)
}
source, err := utils.Abs(matches[0])
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return "", "", errors.Wrapf(err, "Failed to resolve absolute path for '%s'", matches[0])
}
dir, err := utils.DirExists(source)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return "", "", errors.Wrapf(err, "Checking file info '%s' failed", source)
}
if dir {
return pathEnumFolder, source, nil
} else {
return pathEnumArchive, source, nil
}
}
func addConfigTelemetryData(utils cnbutils.BuildUtils, data *cnbBuildTelemetryData, dockerImage string, config *cnbBuildOptions) {
var bindingKeys []string
for k := range config.Bindings {
bindingKeys = append(bindingKeys, k)
}
data.ImageTag = config.ContainerImageTag
data.AdditionalTags = config.AdditionalTags
data.BindingKeys = bindingKeys
data.Path, _, _ = config.resolvePath(utils) // ignore error here, telemetry problems should not fail the build
configKeys := data.BuildEnv.KeysFromConfig
overallKeys := data.BuildEnv.KeysOverall
for key := range config.BuildEnvVars {
configKeys = append(configKeys, key)
overallKeys = append(overallKeys, key)
}
data.BuildEnv.KeysFromConfig = configKeys
data.BuildEnv.KeysOverall = overallKeys
buildTool, _ := getBuildToolFromStageConfig("cnbBuild") // ignore error here, telemetry problems should not fail the build
data.BuildTool = buildTool
data.Buildpacks.FromConfig = privacy.FilterBuildpacks(config.Buildpacks)
data.Builder = privacy.FilterBuilder(dockerImage)
}
func addProjectDescriptorTelemetryData(data *cnbBuildTelemetryData, descriptor project.Descriptor) {
descriptorKeys := data.BuildEnv.KeysFromProjectDescriptor
overallKeys := data.BuildEnv.KeysOverall
for key := range descriptor.EnvVars {
descriptorKeys = append(descriptorKeys, key)
overallKeys = append(overallKeys, key)
}
data.BuildEnv.KeysFromProjectDescriptor = descriptorKeys
data.BuildEnv.KeysOverall = overallKeys
data.Buildpacks.FromProjectDescriptor = privacy.FilterBuildpacks(descriptor.Buildpacks)
data.ProjectDescriptor.Used = true
data.ProjectDescriptor.IncludeUsed = descriptor.Include != nil
data.ProjectDescriptor.ExcludeUsed = descriptor.Exclude != nil
}
func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error {
stepName := "cnbBuild"
cnbTelemetry := &cnbBuildTelemetry{
Version: 3,
}
dockerImage, err := GetDockerImageValue(stepName)
if err != nil {
log.Entry().Warnf("failed to retrieve dockerImage configuration: '%v'", err)
}
cnbTelemetry.dockerImage = dockerImage
cnbBuildConfig := buildsettings.BuildOptions{
CreateBOM: config.CreateBOM,
DockerImage: dockerImage,
BuildSettingsInfo: config.BuildSettingsInfo,
}
log.Entry().Debugf("creating build settings information...")
buildSettingsInfo, err := buildsettings.CreateBuildSettingsInfo(&cnbBuildConfig, stepName)
if err != nil {
log.Entry().Warnf("failed to create build settings info: %v", err)
}
commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo
if len(config.DockerConfigJSON) > 0 {
err = renameDockerConfig(config, utils)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return errors.Wrapf(err, "failed to rename DockerConfigJSON file '%s'", config.DockerConfigJSON)
}
}
if config.ContainerRegistryUser != "" && config.ContainerRegistryPassword != "" {
log.Entry().Debug("enhancing docker config with the provided credentials")
if config.DockerConfigJSON == "" {
config.DockerConfigJSON = "/tmp/config.json"
}
log.Entry().Debugf("using docker config file %q", config.DockerConfigJSON)
_, err = docker.CreateDockerConfigJSON(config.ContainerRegistryURL, config.ContainerRegistryUser, config.ContainerRegistryPassword, "", config.DockerConfigJSON, utils)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "failed to update DockerConfigJSON file %q", config.DockerConfigJSON)
}
log.Entry().Debugf("docker config %q has been updated", config.DockerConfigJSON)
}
mergedConfigs, err := processConfigs(*config, config.MultipleImages)
if err != nil {
return errors.Wrap(err, "failed to process config")
}
for _, c := range mergedConfigs {
err = runCnbBuild(&c, cnbTelemetry, utils, commonPipelineEnvironment, httpClient)
if err != nil {
return err
}
}
if config.CreateBOM {
err = syft.GenerateSBOM(config.SyftDownloadURL, filepath.Dir(config.DockerConfigJSON), utils, utils, httpClient, commonPipelineEnvironment.container.registryURL, commonPipelineEnvironment.container.imageNameTags)
if err != nil {
log.SetErrorCategory(log.ErrorCompliance)
return errors.Wrap(err, "failed to create BOM file")
}
}
telemetryData.Custom1Label = "cnbBuildStepData"
customData, err := json.Marshal(cnbTelemetry)
if err != nil {
return errors.Wrap(err, "failed to marshal custom telemetry data")
}
telemetryData.Custom1 = string(customData)
return nil
}
func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error {
err := cleanDir("/layers", utils)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrap(err, "failed to clean up layers folder /layers")
}
err = cleanDir(platformPath, utils)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrap(err, fmt.Sprintf("failed to clean up platform folder %s", platformPath))
}
tempdir, err := os.MkdirTemp("", "cnbBuild-")
if err != nil {
return errors.Wrap(err, "failed to create tempdir")
}
defer os.RemoveAll(tempdir)
if config.BuildEnvVars == nil {
config.BuildEnvVars = map[string]interface{}{}
}
config.BuildEnvVars["TMPDIR"] = tempdir
customTelemetryData := cnbBuildTelemetryData{}
addConfigTelemetryData(utils, &customTelemetryData, cnbTelemetry.dockerImage, config)
err = isBuilder(utils)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return errors.Wrap(err, "the provided dockerImage is not a valid builder")
}
include := ignore.CompileIgnoreLines("**/*")
exclude := ignore.CompileIgnoreLines("piper", ".pipeline", ".git")
projDescPath, err := project.ResolvePath(config.ProjectDescriptor, config.Path, utils)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return errors.Wrap(err, "failed to check if project descriptor exists")
}
var projectID string
if projDescPath != "" {
descriptor, err := project.ParseDescriptor(projDescPath, utils, httpClient)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return errors.Wrapf(err, "failed to parse %s", projDescPath)
}
addProjectDescriptorTelemetryData(&customTelemetryData, *descriptor)
config.mergeEnvVars(descriptor.EnvVars)
if (config.Buildpacks == nil || len(config.Buildpacks) == 0) && len(descriptor.Buildpacks) > 0 {
config.Buildpacks = descriptor.Buildpacks
}
if descriptor.Exclude != nil {
exclude = descriptor.Exclude
}
if descriptor.Include != nil {
include = descriptor.Include
}
projectID = descriptor.ProjectID
}
targetImage, err := cnbutils.GetTargetImage(config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag, projectID, GeneralConfig.EnvRootPath)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return errors.Wrap(err, "failed to retrieve target image configuration")
}
customTelemetryData.Buildpacks.Overall = privacy.FilterBuildpacks(config.Buildpacks)
customTelemetryData.BuildEnv.KeyValues = privacy.FilterEnv(config.BuildEnvVars)
cnbTelemetry.Data = append(cnbTelemetry.Data, customTelemetryData)
if commonPipelineEnvironment.container.imageNameTag == "" {
commonPipelineEnvironment.container.registryURL = fmt.Sprintf("%s://%s", targetImage.ContainerRegistry.Scheme, targetImage.ContainerRegistry.Host)
commonPipelineEnvironment.container.imageNameTag = fmt.Sprintf("%v:%v", targetImage.ContainerImageName, targetImage.ContainerImageTag)
}
commonPipelineEnvironment.container.imageNameTags = append(commonPipelineEnvironment.container.imageNameTags, fmt.Sprintf("%v:%v", targetImage.ContainerImageName, targetImage.ContainerImageTag))
imageNameAlias := targetImage.ContainerImageName
if config.ContainerImageAlias != "" {
imageNameAlias = config.ContainerImageAlias
}
commonPipelineEnvironment.container.imageNames = append(commonPipelineEnvironment.container.imageNames, imageNameAlias)
if config.BuildEnvVars != nil && len(config.BuildEnvVars) > 0 {
log.Entry().Infof("Setting custom environment variables: '%v'", config.BuildEnvVars)
err = cnbutils.CreateEnvFiles(utils, platformPath, config.BuildEnvVars)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return errors.Wrap(err, "failed to write environment variables to files")
}
}
err = bindings.ProcessBindings(utils, httpClient, platformPath, config.Bindings)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return errors.Wrap(err, "failed process bindings")
}
pathType, source, err := config.resolvePath(utils)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "could not resolve path")
}
target := "/workspace"
err = cleanDir(target, utils)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "failed to clean up target folder %s", target)
}
if pathType != pathEnumArchive {
err = cnbutils.CopyProject(source, target, include, exclude, utils)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target)
}
} else {
err = extractZip(source, target)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target)
}
}
if ok, _ := utils.FileExists(filepath.Join(target, "pom.xml")); ok {
err = linkTargetFolder(utils, source, target)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return err
}
}
metadata.WriteProjectMetadata(GeneralConfig.EnvRootPath, utils)
var buildpacksPath = "/cnb/buildpacks"
var orderPath = "/cnb/order.toml"
if config.Buildpacks != nil && len(config.Buildpacks) > 0 {
log.Entry().Infof("Setting custom buildpacks: '%v'", config.Buildpacks)
buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, config.DockerConfigJSON, utils)
defer func() { _ = utils.RemoveAll(buildpacksPath) }()
defer func() { _ = utils.RemoveAll(orderPath) }()
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "Setting custom buildpacks: %v", config.Buildpacks)
}
}
cnbRegistryAuth, err := cnbutils.GenerateCnbAuth(config.DockerConfigJSON, utils)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return errors.Wrap(err, "failed to generate CNB_REGISTRY_AUTH")
}
if len(config.CustomTLSCertificateLinks) > 0 {
caCertificates := "/tmp/ca-certificates.crt"
_, err := utils.Copy("/etc/ssl/certs/ca-certificates.crt", caCertificates)
if err != nil {
return errors.Wrap(err, "failed to copy certificates")
}
err = certutils.CertificateUpdate(config.CustomTLSCertificateLinks, httpClient, utils, caCertificates)
if err != nil {
return errors.Wrap(err, "failed to update certificates")
}
utils.AppendEnv([]string{fmt.Sprintf("SSL_CERT_FILE=%s", caCertificates)})
} else {
log.Entry().Info("skipping certificates update")
}
utils.AppendEnv([]string{fmt.Sprintf("CNB_REGISTRY_AUTH=%s", cnbRegistryAuth)})
utils.AppendEnv([]string{"CNB_PLATFORM_API=0.9"})
creatorArgs := []string{
"-no-color",
"-buildpacks", buildpacksPath,
"-order", orderPath,
"-platform", platformPath,
"-skip-restore",
}
if GeneralConfig.Verbose {
creatorArgs = append(creatorArgs, "-log-level", "debug")
}
if config.RunImage != "" {
creatorArgs = append(creatorArgs, "-run-image", config.RunImage)
}
if config.DefaultProcess != "" {
creatorArgs = append(creatorArgs, "-process-type", config.DefaultProcess)
}
containerImage := path.Join(targetImage.ContainerRegistry.Host, targetImage.ContainerImageName)
for _, tag := range config.AdditionalTags {
target := fmt.Sprintf("%s:%s", containerImage, tag)
if !piperutils.ContainsString(creatorArgs, target) {
creatorArgs = append(creatorArgs, "-tag", target)
}
}
creatorArgs = append(creatorArgs, fmt.Sprintf("%s:%s", containerImage, targetImage.ContainerImageTag))
err = utils.RunExecutable(creatorPath, creatorArgs...)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "execution of '%s' failed", creatorArgs)
}
digest, err := cnbutils.DigestFromReport(utils)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrap(err, "failed to read image digest")
}
commonPipelineEnvironment.container.imageDigest = digest
commonPipelineEnvironment.container.imageDigests = append(commonPipelineEnvironment.container.imageDigests, digest)
if len(config.PreserveFiles) > 0 {
if pathType != pathEnumArchive {
err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils)
if err != nil {
log.SetErrorCategory(log.ErrorBuild)
return errors.Wrapf(err, "failed to preserve files using glob '%s'", config.PreserveFiles)
}
} else {
log.Entry().Warnf("skipping preserving files because the source '%s' is an archive", source)
}
}
return nil
}