From 4ab32f7e7c1738a3d6b95729bff3a936bf4c8ec3 Mon Sep 17 00:00:00 2001 From: Pavel Busko Date: Tue, 14 Jun 2022 15:14:44 +0200 Subject: [PATCH] feat(cnbBuild): generate buildSettingInfo Co-authored-by: Sumit Kulhadia Co-authored-by: Johannes Dillmann --- cmd/cnbBuild.go | 48 ++++++++++++++++--------- cmd/cnbBuild_generated.go | 21 +++++++++++ cmd/cnbBuild_test.go | 35 +++++++----------- pkg/buildsettings/buildSettings.go | 5 +++ pkg/buildsettings/buildSettings_test.go | 5 +++ resources/metadata/cnbBuild.yaml | 11 ++++++ 6 files changed, 86 insertions(+), 39 deletions(-) diff --git a/cmd/cnbBuild.go b/cmd/cnbBuild.go index fcdc3c999..80e9d8b0b 100644 --- a/cmd/cnbBuild.go +++ b/cmd/cnbBuild.go @@ -8,6 +8,7 @@ import ( "path" "path/filepath" + "github.com/SAP/jenkins-library/pkg/buildsettings" "github.com/SAP/jenkins-library/pkg/certutils" "github.com/SAP/jenkins-library/pkg/cnbutils" "github.com/SAP/jenkins-library/pkg/cnbutils/bindings" @@ -47,8 +48,9 @@ type cnbBuildUtilsBundle struct { } type cnbBuildTelemetry struct { - Version int `json:"version"` - Data []cnbBuildTelemetryData `json:"data"` + dockerImage string + Version int `json:"version"` + Data []cnbBuildTelemetryData `json:"data"` } type cnbBuildTelemetryData struct { @@ -306,7 +308,7 @@ func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (pathEnum, } } -func addConfigTelemetryData(utils cnbutils.BuildUtils, data *cnbBuildTelemetryData, config *cnbBuildOptions) { +func addConfigTelemetryData(utils cnbutils.BuildUtils, data *cnbBuildTelemetryData, dockerImage string, config *cnbBuildOptions) { var bindingKeys []string for k := range config.Bindings { bindingKeys = append(bindingKeys, k) @@ -330,13 +332,7 @@ func addConfigTelemetryData(utils cnbutils.BuildUtils, data *cnbBuildTelemetryDa data.Buildpacks.FromConfig = privacy.FilterBuildpacks(config.Buildpacks) - dockerImage, err := GetDockerImageValue("cnbBuild") - if err != nil { - log.Entry().Warnf("Error while preparing telemetry: retrieving docker image failed: '%v'", err) - data.Builder = "" - } else { - data.Builder = privacy.FilterBuilder(dockerImage) - } + data.Builder = privacy.FilterBuilder(dockerImage) } func addProjectDescriptorTelemetryData(data *cnbBuildTelemetryData, descriptor project.Descriptor) { @@ -357,22 +353,42 @@ func addProjectDescriptorTelemetryData(data *cnbBuildTelemetryData, descriptor p } func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { - telemetry := &cnbBuildTelemetry{ + stepName := "cnbBuild" + cnbTelemetry := &cnbBuildTelemetry{ Version: 3, } + + dockerImage, err := GetDockerImageValue(stepName) + if err != nil { + log.Entry().Warnf("failed to retrieve dockerImage configuration: '%v'", err) + } + + cnbTelemetry.dockerImage = dockerImage + + cnbBuildConfig := buildsettings.BuildOptions{ + DockerImage: dockerImage, + BuildSettingsInfo: config.BuildSettingsInfo, + } + log.Entry().Debugf("creating build settings information...") + buildSettingsInfo, err := buildsettings.CreateBuildSettingsInfo(&cnbBuildConfig, stepName) + if err != nil { + log.Entry().Warnf("failed to create build settings info: %v", err) + } + commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo + mergedConfigs, err := processConfigs(*config, config.MultipleImages) if err != nil { return errors.Wrap(err, "failed to process config") } for _, c := range mergedConfigs { - err = runCnbBuild(&c, telemetryData, telemetry, utils, commonPipelineEnvironment, httpClient) + err = runCnbBuild(&c, cnbTelemetry, utils, commonPipelineEnvironment, httpClient) if err != nil { return err } } telemetryData.Custom1Label = "cnbBuildStepData" - customData, err := json.Marshal(telemetry) + customData, err := json.Marshal(cnbTelemetry) if err != nil { return errors.Wrap(err, "failed to marshal custom telemetry data") } @@ -380,7 +396,7 @@ func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, return nil } -func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, telemetry *cnbBuildTelemetry, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { +func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { err := cleanDir("/layers", utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) @@ -394,7 +410,7 @@ func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, t } customTelemetryData := cnbBuildTelemetryData{} - addConfigTelemetryData(utils, &customTelemetryData, config) + addConfigTelemetryData(utils, &customTelemetryData, cnbTelemetry.dockerImage, config) err = isBuilder(utils) if err != nil { @@ -444,7 +460,7 @@ func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, t } customTelemetryData.Buildpacks.Overall = privacy.FilterBuildpacks(config.Buildpacks) customTelemetryData.BuildEnv.KeyValues = privacy.FilterEnv(config.BuildEnvVars) - telemetry.Data = append(telemetry.Data, customTelemetryData) + cnbTelemetry.Data = append(cnbTelemetry.Data, customTelemetryData) if commonPipelineEnvironment.container.imageNameTag == "" { commonPipelineEnvironment.container.registryURL = fmt.Sprintf("%s://%s", targetImage.ContainerRegistry.Scheme, targetImage.ContainerRegistry.Host) diff --git a/cmd/cnbBuild_generated.go b/cmd/cnbBuild_generated.go index 7d04f3f64..babae172a 100644 --- a/cmd/cnbBuild_generated.go +++ b/cmd/cnbBuild_generated.go @@ -31,6 +31,7 @@ type cnbBuildOptions struct { Bindings map[string]interface{} `json:"bindings,omitempty"` MultipleImages []map[string]interface{} `json:"multipleImages,omitempty"` PreserveFiles []string `json:"preserveFiles,omitempty"` + BuildSettingsInfo string `json:"buildSettingsInfo,omitempty"` } type cnbBuildCommonPipelineEnvironment struct { @@ -42,6 +43,9 @@ type cnbBuildCommonPipelineEnvironment struct { imageNameTags []string imageDigests []string } + custom struct { + buildSettingsInfo string + } } func (p *cnbBuildCommonPipelineEnvironment) persist(path, resourceName string) { @@ -56,6 +60,7 @@ func (p *cnbBuildCommonPipelineEnvironment) persist(path, resourceName string) { {category: "container", name: "imageNames", value: p.container.imageNames}, {category: "container", name: "imageNameTags", value: p.container.imageNameTags}, {category: "container", name: "imageDigests", value: p.container.imageDigests}, + {category: "custom", name: "buildSettingsInfo", value: p.custom.buildSettingsInfo}, } errCount := 0 @@ -176,6 +181,7 @@ func addCnbBuildFlags(cmd *cobra.Command, stepConfig *cnbBuildOptions) { cmd.Flags().StringSliceVar(&stepConfig.AdditionalTags, "additionalTags", []string{}, "List of tags which will be pushed to the registry (additionally to the provided `containerImageTag`), e.g. \"latest\".") cmd.Flags().StringSliceVar(&stepConfig.PreserveFiles, "preserveFiles", []string{}, "List of globs, for keeping build results in the Jenkins workspace.\n\n*Note*: globs will be calculated relative to the [path](#path) property.\n") + cmd.Flags().StringVar(&stepConfig.BuildSettingsInfo, "buildSettingsInfo", os.Getenv("PIPER_buildSettingsInfo"), "Build settings info is typically filled by the step automatically to create information about the build settings that were used during the mta build. This information is typically used for compliance related processes.") cmd.MarkFlagRequired("containerImageTag") cmd.MarkFlagRequired("containerRegistryUrl") @@ -345,6 +351,20 @@ func cnbBuildMetadata() config.StepData { Aliases: []config.Alias{}, Default: []string{}, }, + { + Name: "buildSettingsInfo", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "custom/buildSettingsInfo", + }, + }, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_buildSettingsInfo"), + }, }, }, Containers: []config.Container{ @@ -362,6 +382,7 @@ func cnbBuildMetadata() config.StepData { {"name": "container/imageNames", "type": "[]string"}, {"name": "container/imageNameTags", "type": "[]string"}, {"name": "container/imageDigests", "type": "[]string"}, + {"name": "custom/buildSettingsInfo"}, }, }, }, diff --git a/cmd/cnbBuild_test.go b/cmd/cnbBuild_test.go index 7a2970442..25cace7cf 100644 --- a/cmd/cnbBuild_test.go +++ b/cmd/cnbBuild_test.go @@ -74,6 +74,7 @@ func TestRunCnbBuild(t *testing.T) { assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:%s", imageRegistry, config.ContainerImageName, config.ContainerImageTag)) assert.Equal(t, config.ContainerRegistryURL, commonPipelineEnvironment.container.registryURL) assert.Equal(t, "my-image:0.0.1", commonPipelineEnvironment.container.imageNameTag) + assert.Equal(t, `{"cnbBuild":[{"dockerImage":"paketobuildpacks/builder:base"}]}`, commonPipelineEnvironment.custom.buildSettingsInfo) }) t.Run("prefers project descriptor", func(t *testing.T) { @@ -169,7 +170,6 @@ func TestRunCnbBuild(t *testing.T) { t.Run("success case (custom buildpacks and custom env variables, renaming docker conf file, additional tag)", func(t *testing.T) { t.Parallel() - commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} config := cnbBuildOptions{ ContainerImageName: "my-image", ContainerImageTag: "0.0.1", @@ -186,7 +186,7 @@ func TestRunCnbBuild(t *testing.T) { utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`)) addBuilderFiles(&utils) - err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) require.NoError(t, err) runner := utils.ExecMockRunner @@ -206,7 +206,6 @@ func TestRunCnbBuild(t *testing.T) { t.Run("success case (customTlsCertificates)", func(t *testing.T) { t.Parallel() - commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} httpmock.Activate() defer httpmock.DeactivateAndReset() httpmock.RegisterResponder(http.MethodGet, "https://test-cert.com/cert.crt", httpmock.NewStringResponder(200, "testCert")) @@ -229,7 +228,7 @@ func TestRunCnbBuild(t *testing.T) { utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`)) addBuilderFiles(&utils) - err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, client) + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, client) require.NoError(t, err) result, err := utils.FilesMock.FileRead(caCertsTmpFile) @@ -246,7 +245,6 @@ func TestRunCnbBuild(t *testing.T) { t.Run("success case (additionalTags)", func(t *testing.T) { t.Parallel() - commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} config := cnbBuildOptions{ ContainerImageName: "my-image", ContainerImageTag: "3.1.5", @@ -259,7 +257,7 @@ func TestRunCnbBuild(t *testing.T) { utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`)) addBuilderFiles(&utils) - err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) require.NoError(t, err) runner := utils.ExecMockRunner @@ -321,7 +319,6 @@ func TestRunCnbBuild(t *testing.T) { t.Run("error case: Invalid DockerConfigJSON file", func(t *testing.T) { t.Parallel() - commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} config := cnbBuildOptions{ ContainerImageTag: "0.0.1", ContainerRegistryURL: imageRegistry, @@ -333,13 +330,12 @@ func TestRunCnbBuild(t *testing.T) { utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":"dXNlcjpwYXNz"}}`)) addBuilderFiles(&utils) - err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) assert.EqualError(t, err, "failed to generate CNB_REGISTRY_AUTH: json: cannot unmarshal string into Go struct field ConfigFile.auths of type types.AuthConfig") }) t.Run("error case: DockerConfigJSON file not there (config.json)", func(t *testing.T) { t.Parallel() - commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} config := cnbBuildOptions{ ContainerImageTag: "0.0.1", ContainerRegistryURL: imageRegistry, @@ -350,13 +346,12 @@ func TestRunCnbBuild(t *testing.T) { utils := newCnbBuildTestsUtils() addBuilderFiles(&utils) - err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) assert.EqualError(t, err, "failed to generate CNB_REGISTRY_AUTH: could not read 'not-there/config.json'") }) t.Run("error case: DockerConfigJSON file not there (not config.json)", func(t *testing.T) { t.Parallel() - commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} config := cnbBuildOptions{ ContainerImageTag: "0.0.1", ContainerRegistryURL: imageRegistry, @@ -367,24 +362,22 @@ func TestRunCnbBuild(t *testing.T) { utils := newCnbBuildTestsUtils() addBuilderFiles(&utils) - err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) assert.EqualError(t, err, "failed to rename DockerConfigJSON file 'not-there': renaming file 'not-there' is not supported, since it does not exist, or is not a leaf-entry") }) t.Run("error case: dockerImage is not a valid builder", func(t *testing.T) { t.Parallel() - commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} config := cnbBuildOptions{} utils := newCnbBuildTestsUtils() - err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) assert.EqualError(t, err, "the provided dockerImage is not a valid builder: binary '/cnb/lifecycle/creator' not found") }) t.Run("error case: builder image does not contain tls certificates", func(t *testing.T) { t.Parallel() - commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} config := cnbBuildOptions{ ContainerImageName: "my-image", ContainerImageTag: "0.0.1", @@ -398,13 +391,12 @@ func TestRunCnbBuild(t *testing.T) { utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`)) addBuilderFiles(&utils) - err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) assert.EqualError(t, err, "failed to copy certificates: cannot copy '/etc/ssl/certs/ca-certificates.crt': file does not exist") }) t.Run("success case (telemetry was added)", func(t *testing.T) { t.Parallel() - commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} registry := "some-registry" config := cnbBuildOptions{ ContainerImageName: "my-image", @@ -438,7 +430,7 @@ uri = "some-buildpack"`)) addBuilderFiles(&utils) telemetryData := telemetry.CustomData{} - err := callCnbBuild(&config, &telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + err := callCnbBuild(&config, &telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) require.NoError(t, err) customDataAsString := telemetryData.Custom1 @@ -467,7 +459,6 @@ uri = "some-buildpack"`)) t.Run("error case, multiple artifacts in path", func(t *testing.T) { t.Parallel() - commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} config := cnbBuildOptions{ ContainerImageName: "my-image", ContainerImageTag: "3.1.5", @@ -488,7 +479,7 @@ uri = "some-buildpack"`)) addBuilderFiles(&utils) telemetryData := telemetry.CustomData{} - err := callCnbBuild(&config, &telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + err := callCnbBuild(&config, &telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) require.EqualError(t, err, "could not resolve path: Failed to resolve glob for 'target/*.jar', matching 2 file(s)") }) @@ -526,7 +517,6 @@ uri = "some-buildpack"`)) t.Run("success case (build env telemetry was added)", func(t *testing.T) { t.Parallel() - commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} registry := "some-registry" config := cnbBuildOptions{ ContainerImageName: "my-image", @@ -558,7 +548,7 @@ uri = "some-buildpack" addBuilderFiles(&utils) telemetryData := telemetry.CustomData{} - err := callCnbBuild(&config, &telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + err := callCnbBuild(&config, &telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) require.NoError(t, err) customDataAsString := telemetryData.Custom1 @@ -623,5 +613,4 @@ uri = "some-buildpack" assert.Equal(t, "my-image-0:3.1.5", commonPipelineEnvironment.container.imageNameTag) }) - } diff --git a/pkg/buildsettings/buildSettings.go b/pkg/buildsettings/buildSettings.go index ca05eabd7..6b66dbc34 100644 --- a/pkg/buildsettings/buildSettings.go +++ b/pkg/buildsettings/buildSettings.go @@ -17,6 +17,7 @@ type BuildSettings struct { MtaBuild []BuildOptions `json:"mtaBuild,omitempty"` PythonBuild []BuildOptions `json:"pythonBuild,omitempty"` NpmExecuteScripts []BuildOptions `json:"npmExecuteScripts,omitempty"` + CnbBuild []BuildOptions `json:"cnbBuild,omitempty"` } type BuildOptions struct { @@ -101,6 +102,10 @@ func CreateBuildSettingsInfo(config *BuildOptions, buildTool string) (string, er jsonResult, err = json.Marshal(BuildSettings{ NpmExecuteScripts: settings, }) + case "cnbBuild": + jsonResult, err = json.Marshal(BuildSettings{ + CnbBuild: settings, + }) default: log.Entry().Warningf("buildTool '%s' not supported for creation of build settings", buildTool) return "", nil diff --git a/pkg/buildsettings/buildSettings_test.go b/pkg/buildsettings/buildSettings_test.go index 5fbfde1f6..24988df9b 100644 --- a/pkg/buildsettings/buildSettings_test.go +++ b/pkg/buildsettings/buildSettings_test.go @@ -64,6 +64,11 @@ func TestCreateBuildSettingsInfo(t *testing.T) { buildTool: "npmExecuteScripts", expected: "{\"npmExecuteScripts\":[{\"createBOM\":true}]}", }, + { + config: BuildOptions{DockerImage: "builder:latest"}, + buildTool: "cnbBuild", + expected: "{\"cnbBuild\":[{\"dockerImage\":\"builder:latest\"}]}", + }, } for _, testCase := range testTableConfig { diff --git a/resources/metadata/cnbBuild.yaml b/resources/metadata/cnbBuild.yaml index 688c73bbc..a5ef04015 100644 --- a/resources/metadata/cnbBuild.yaml +++ b/resources/metadata/cnbBuild.yaml @@ -233,6 +233,16 @@ spec: - PARAMETERS - STAGES - STEPS + - name: buildSettingsInfo + type: string + description: Build settings info is typically filled by the step automatically to create information about the build settings that were used during the mta build. This information is typically used for compliance related processes. + scope: + - STEPS + - STAGES + - PARAMETERS + resourceRef: + - name: commonPipelineEnvironment + param: custom/buildSettingsInfo outputs: resources: - name: commonPipelineEnvironment @@ -247,5 +257,6 @@ spec: type: "[]string" - name: container/imageDigests type: "[]string" + - name: custom/buildSettingsInfo containers: - image: "paketobuildpacks/builder:base"