package cmd import ( "archive/zip" "fmt" "os" "path" "path/filepath" "github.com/SAP/jenkins-library/pkg/buildpacks" "github.com/SAP/jenkins-library/pkg/buildsettings" "github.com/SAP/jenkins-library/pkg/certutils" "github.com/SAP/jenkins-library/pkg/cnbutils" "github.com/SAP/jenkins-library/pkg/cnbutils/bindings" "github.com/SAP/jenkins-library/pkg/cnbutils/project" "github.com/SAP/jenkins-library/pkg/cnbutils/project/metadata" "github.com/SAP/jenkins-library/pkg/command" "github.com/SAP/jenkins-library/pkg/docker" piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/syft" "github.com/SAP/jenkins-library/pkg/telemetry" "github.com/imdario/mergo" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ignore "github.com/sabhiram/go-gitignore" ) const ( creatorPath = "/cnb/lifecycle/creator" platformPath = "/tmp/platform" platformAPIVersion = "0.11" ) type cnbBuildUtilsBundle struct { *command.Command *piperutils.Files *docker.Client } func processConfigs(main cnbBuildOptions, multipleImages []map[string]interface{}) ([]cnbBuildOptions, error) { var result []cnbBuildOptions if len(multipleImages) == 0 { result = append(result, main) return result, nil } for _, conf := range multipleImages { var structuredConf cnbBuildOptions err := mapstructure.Decode(conf, &structuredConf) if err != nil { return nil, err } err = mergo.Merge(&structuredConf, main) if err != nil { return nil, err } result = append(result, structuredConf) } return result, nil } func setCustomBuildpacks(bpacks, preBuildpacks, postBuildpacks []string, dockerCreds string, utils cnbutils.BuildUtils) (string, string, error) { buildpacksPath := "/tmp/buildpacks" orderPath := "/tmp/buildpacks/order.toml" err := cnbutils.DownloadBuildpacks(buildpacksPath, append(bpacks, append(preBuildpacks, postBuildpacks...)...), dockerCreds, utils) if err != nil { return "", "", err } if len(bpacks) == 0 && (len(postBuildpacks) > 0 || len(preBuildpacks) > 0) { matches, err := utils.Glob("/cnb/buildpacks/*") if err != nil { return "", "", err } for _, match := range matches { err = cnbutils.CreateVersionSymlinks(buildpacksPath, match, utils) if err != nil { return "", "", err } } } newOrder, err := cnbutils.CreateOrder(bpacks, preBuildpacks, postBuildpacks, dockerCreds, utils) if err != nil { return "", "", err } err = newOrder.Save(orderPath) if err != nil { return "", "", err } return buildpacksPath, orderPath, nil } func newCnbBuildUtils() cnbutils.BuildUtils { utils := cnbBuildUtilsBundle{ Command: &command.Command{ StepName: "cnbBuild", }, Files: &piperutils.Files{}, Client: &docker.Client{}, } utils.Stdout(log.Writer()) utils.Stderr(log.Writer()) return &utils } func cnbBuild(config cnbBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment) { utils := newCnbBuildUtils() client := &piperhttp.Client{} err := callCnbBuild(&config, telemetryData, utils, commonPipelineEnvironment, client) if err != nil { log.Entry().WithError(err).Fatal("step execution failed") } } func isBuilder(utils cnbutils.BuildUtils) error { exists, err := utils.FileExists(creatorPath) if err != nil { return err } if !exists { return fmt.Errorf("binary '%s' not found", creatorPath) } return nil } func isZip(path string) bool { r, err := zip.OpenReader(path) switch { case err == nil: _ = r.Close() return true case err == zip.ErrFormat: return false default: return false } } func cleanDir(dir string, utils cnbutils.BuildUtils) error { dirContent, err := utils.Glob(filepath.Join(dir, "*")) if err != nil { return err } for _, obj := range dirContent { err = utils.RemoveAll(obj) if err != nil { return err } } return nil } func extractZip(source, target string) error { if isZip(source) { log.Entry().Infof("Extracting archive '%s' to '%s'", source, target) _, err := piperutils.Unzip(source, target) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrapf(err, "Extracting archive '%s' to '%s' failed", source, target) } } else { log.SetErrorCategory(log.ErrorBuild) return errors.New("application path must be a directory or zip") } return nil } func ensureDockerConfig(config *cnbBuildOptions, utils cnbutils.BuildUtils) error { newFile := "/tmp/config.json" if config.DockerConfigJSON == "" { config.DockerConfigJSON = newFile return utils.FileWrite(config.DockerConfigJSON, []byte("{}"), os.ModePerm) } log.Entry().Debugf("Copying docker config file from '%s' to '%s'", config.DockerConfigJSON, newFile) _, err := utils.Copy(config.DockerConfigJSON, newFile) if err != nil { return err } err = utils.Chmod(newFile, 0644) if err != nil { return err } config.DockerConfigJSON = newFile return nil } func linkTargetFolder(utils cnbutils.BuildUtils, source, target string) error { var err error linkPath := filepath.Join(target, "target") targetPath := filepath.Join(source, "target") if ok, _ := utils.DirExists(targetPath); !ok { err = utils.MkdirAll(targetPath, os.ModePerm) if err != nil { return err } } if ok, _ := utils.DirExists(linkPath); ok { err = utils.RemoveAll(linkPath) if err != nil { return err } } return utils.Symlink(targetPath, linkPath) } func (config *cnbBuildOptions) mergeEnvVars(vars map[string]interface{}) { if config.BuildEnvVars == nil { config.BuildEnvVars = vars return } for k, v := range vars { _, exists := config.BuildEnvVars[k] if !exists { config.BuildEnvVars[k] = v } } } func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (buildpacks.PathEnum, string, error) { pwd, err := utils.Getwd() if err != nil { log.SetErrorCategory(log.ErrorBuild) return "", "", errors.Wrap(err, "failed to get current working directory") } if config.Path == "" { return buildpacks.PathEnumRoot, pwd, nil } matches, err := utils.Glob(config.Path) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return "", "", errors.Wrapf(err, "Failed to resolve glob for '%s'", config.Path) } numMatches := len(matches) if numMatches != 1 { log.SetErrorCategory(log.ErrorConfiguration) return "", "", errors.Errorf("Failed to resolve glob for '%s', matching %d file(s)", config.Path, numMatches) } source, err := utils.Abs(matches[0]) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return "", "", errors.Wrapf(err, "Failed to resolve absolute path for '%s'", matches[0]) } dir, err := utils.DirExists(source) if err != nil { log.SetErrorCategory(log.ErrorBuild) return "", "", errors.Wrapf(err, "Checking file info '%s' failed", source) } if dir { return buildpacks.PathEnumFolder, source, nil } else { return buildpacks.PathEnumArchive, source, nil } } func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { stepName := "cnbBuild" telemetry := buildpacks.NewTelemetry(telemetryData) dockerImage, err := GetDockerImageValue(stepName) if err != nil { log.Entry().Warnf("failed to retrieve dockerImage configuration: '%v'", err) } telemetry.WithImage(dockerImage) cnbBuildConfig := buildsettings.BuildOptions{ CreateBOM: config.CreateBOM, DockerImage: dockerImage, BuildSettingsInfo: config.BuildSettingsInfo, } log.Entry().Debugf("creating build settings information...") buildSettingsInfo, err := buildsettings.CreateBuildSettingsInfo(&cnbBuildConfig, stepName) if err != nil { log.Entry().Warnf("failed to create build settings info: %v", err) } commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo err = ensureDockerConfig(config, utils) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrapf(err, "failed to create/rename DockerConfigJSON file") } if config.DockerConfigJSONCPE != "" { log.Entry().Debugf("merging docker config file '%s' into '%s'", config.DockerConfigJSONCPE, config.DockerConfigJSON) err = docker.MergeDockerConfigJSON(config.DockerConfigJSONCPE, config.DockerConfigJSON, utils) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrapf(err, "failed to merge DockerConfigJSON files") } } mergedConfigs, err := processConfigs(*config, config.MultipleImages) if err != nil { return errors.Wrap(err, "failed to process config") } buildSummary := cnbutils.NewBuildSummary(dockerImage, utils) for _, c := range mergedConfigs { imageSummary := &cnbutils.ImageSummary{} err = runCnbBuild(&c, telemetry, imageSummary, utils, commonPipelineEnvironment, httpClient) if err != nil { return err } buildSummary.Images = append(buildSummary.Images, imageSummary) } buildSummary.Print() if config.CreateBOM { err = syft.GenerateSBOM(config.SyftDownloadURL, filepath.Dir(config.DockerConfigJSON), utils, utils, httpClient, commonPipelineEnvironment.container.registryURL, commonPipelineEnvironment.container.imageNameTags) if err != nil { log.SetErrorCategory(log.ErrorCompliance) return errors.Wrap(err, "failed to create BOM file") } } return telemetry.Export() } func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, imageSummary *cnbutils.ImageSummary, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { err := cleanDir("/layers", utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrap(err, "failed to clean up layers folder /layers") } err = cleanDir(platformPath, utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrap(err, fmt.Sprintf("failed to clean up platform folder %s", platformPath)) } tempdir, err := os.MkdirTemp("", "cnbBuild-") if err != nil { return errors.Wrap(err, "failed to create tempdir") } defer os.RemoveAll(tempdir) if config.BuildEnvVars == nil { config.BuildEnvVars = map[string]interface{}{} } config.BuildEnvVars["TMPDIR"] = tempdir telemetrySegment := createInitialTelemetrySegment(config, utils) err = isBuilder(utils) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrap(err, "the provided dockerImage is not a valid builder") } include := ignore.CompileIgnoreLines("**/*") exclude := ignore.CompileIgnoreLines("piper", ".pipeline", ".git") projDescPath, err := project.ResolvePath(config.ProjectDescriptor, config.Path, utils) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrap(err, "failed to check if project descriptor exists") } imageSummary.ProjectDescriptor = projDescPath var projectID string if projDescPath != "" { descriptor, err := project.ParseDescriptor(projDescPath, utils, httpClient) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrapf(err, "failed to parse %s", projDescPath) } telemetrySegment.WithProjectDescriptor(descriptor) config.mergeEnvVars(descriptor.EnvVars) if len(config.Buildpacks) == 0 { config.Buildpacks = descriptor.Buildpacks } if len(config.PreBuildpacks) == 0 { config.PreBuildpacks = descriptor.PreBuildpacks } if len(config.PostBuildpacks) == 0 { config.PostBuildpacks = descriptor.PostBuildpacks } if descriptor.Exclude != nil { exclude = descriptor.Exclude } if descriptor.Include != nil { include = descriptor.Include } projectID = descriptor.ProjectID } targetImage, err := cnbutils.GetTargetImage(config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag, projectID, GeneralConfig.EnvRootPath) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrap(err, "failed to retrieve target image configuration") } telemetry.AddSegment(telemetrySegment.WithBuildpacksOverall(config.Buildpacks).WithKeyValues(config.BuildEnvVars)) if commonPipelineEnvironment.container.imageNameTag == "" { commonPipelineEnvironment.container.registryURL = fmt.Sprintf("%s://%s", targetImage.ContainerRegistry.Scheme, targetImage.ContainerRegistry.Host) commonPipelineEnvironment.container.imageNameTag = fmt.Sprintf("%v:%v", targetImage.ContainerImageName, targetImage.ContainerImageTag) } commonPipelineEnvironment.container.imageNameTags = append(commonPipelineEnvironment.container.imageNameTags, fmt.Sprintf("%v:%v", targetImage.ContainerImageName, targetImage.ContainerImageTag)) imageNameAlias := targetImage.ContainerImageName if config.ContainerImageAlias != "" { imageNameAlias = config.ContainerImageAlias } commonPipelineEnvironment.container.imageNames = append(commonPipelineEnvironment.container.imageNames, imageNameAlias) if config.BuildEnvVars != nil && len(config.BuildEnvVars) > 0 { log.Entry().Infof("Setting custom environment variables: '%v'", config.BuildEnvVars) imageSummary.AddEnv(config.BuildEnvVars) err = cnbutils.CreateEnvFiles(utils, platformPath, config.BuildEnvVars) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrap(err, "failed to write environment variables to files") } } err = bindings.ProcessBindings(utils, httpClient, platformPath, config.Bindings) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrap(err, "failed process bindings") } pathType, source, err := config.resolvePath(utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrapf(err, "could not resolve path") } target := "/workspace" err = cleanDir(target, utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrapf(err, "failed to clean up target folder %s", target) } if pathType != buildpacks.PathEnumArchive { err = cnbutils.CopyProject(source, target, include, exclude, utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target) } } else { err = extractZip(source, target) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target) } } if ok, _ := utils.FileExists(filepath.Join(target, "pom.xml")); ok { err = linkTargetFolder(utils, source, target) if err != nil { log.SetErrorCategory(log.ErrorBuild) return err } } metadata.WriteProjectMetadata(GeneralConfig.EnvRootPath, utils) var buildpacksPath = "/cnb/buildpacks" var orderPath = cnbutils.DefaultOrderPath if len(config.Buildpacks) > 0 || len(config.PreBuildpacks) > 0 || len(config.PostBuildpacks) > 0 { log.Entry().Infof("Setting custom buildpacks: '%v'", config.Buildpacks) log.Entry().Infof("Pre-buildpacks: '%v'", config.PreBuildpacks) log.Entry().Infof("Post-buildpacks: '%v'", config.PostBuildpacks) buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, config.PreBuildpacks, config.PostBuildpacks, config.DockerConfigJSON, utils) defer func() { _ = utils.RemoveAll(buildpacksPath) }() defer func() { _ = utils.RemoveAll(orderPath) }() if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrapf(err, "Setting custom buildpacks: %v", config.Buildpacks) } } cnbRegistryAuth, err := cnbutils.GenerateCnbAuth(config.DockerConfigJSON, utils) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrap(err, "failed to generate CNB_REGISTRY_AUTH") } if len(config.CustomTLSCertificateLinks) > 0 { caCertificates := "/tmp/ca-certificates.crt" _, err := utils.Copy("/etc/ssl/certs/ca-certificates.crt", caCertificates) if err != nil { return errors.Wrap(err, "failed to copy certificates") } err = certutils.CertificateUpdate(config.CustomTLSCertificateLinks, httpClient, utils, caCertificates) if err != nil { return errors.Wrap(err, "failed to update certificates") } utils.AppendEnv([]string{fmt.Sprintf("SSL_CERT_FILE=%s", caCertificates)}) } else { log.Entry().Info("skipping certificates update") } utils.AppendEnv([]string{fmt.Sprintf("CNB_REGISTRY_AUTH=%s", cnbRegistryAuth)}) utils.AppendEnv([]string{fmt.Sprintf("CNB_PLATFORM_API=%s", platformAPIVersion)}) creatorArgs := []string{ "-no-color", "-buildpacks", buildpacksPath, "-order", orderPath, "-platform", platformPath, "-skip-restore", } if GeneralConfig.Verbose { creatorArgs = append(creatorArgs, "-log-level", "debug") } if config.RunImage != "" { creatorArgs = append(creatorArgs, "-run-image", config.RunImage) } if config.DefaultProcess != "" { creatorArgs = append(creatorArgs, "-process-type", config.DefaultProcess) } containerImage := path.Join(targetImage.ContainerRegistry.Host, targetImage.ContainerImageName) for _, tag := range config.AdditionalTags { target := fmt.Sprintf("%s:%s", containerImage, tag) if !piperutils.ContainsString(creatorArgs, target) { creatorArgs = append(creatorArgs, "-tag", target) } } creatorArgs = append(creatorArgs, fmt.Sprintf("%s:%s", containerImage, targetImage.ContainerImageTag)) err = utils.RunExecutable(creatorPath, creatorArgs...) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrapf(err, "execution of '%s' failed", creatorArgs) } digest, err := cnbutils.DigestFromReport(utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrap(err, "failed to read image digest") } commonPipelineEnvironment.container.imageDigest = digest commonPipelineEnvironment.container.imageDigests = append(commonPipelineEnvironment.container.imageDigests, digest) imageSummary.ImageRef = fmt.Sprintf("%s@%s", containerImage, digest) if len(config.PreserveFiles) > 0 { if pathType != buildpacks.PathEnumArchive { err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrapf(err, "failed to preserve files using glob '%s'", config.PreserveFiles) } } else { log.Entry().Warnf("skipping preserving files because the source '%s' is an archive", source) } } return nil } func createInitialTelemetrySegment(config *cnbBuildOptions, utils cnbutils.BuildUtils) *buildpacks.Segment { telemetrySegment := buildpacks.NewSegment() projectPath, _, _ := config.resolvePath(utils) // ignore error here, telemetry problems should not fail the build buildTool, _ := getBuildToolFromStageConfig("cnbBuild") // ignore error here, telemetry problems should not fail the build return telemetrySegment.WithBindings(config.Bindings). WithTags(config.ContainerImageTag, config.AdditionalTags). WithPath(projectPath). WithEnv(config.BuildEnvVars). WithBuildTool(buildTool). WithBuildpacksFromConfig(config.Buildpacks) }