diff --git a/cmd/cnbBuild.go b/cmd/cnbBuild.go index 9c80aa8ab..a7347b6f8 100644 --- a/cmd/cnbBuild.go +++ b/cmd/cnbBuild.go @@ -2,17 +2,16 @@ package cmd import ( "archive/zip" - "encoding/json" "fmt" "os" "path" "path/filepath" + "github.com/SAP/jenkins-library/pkg/buildpacks" "github.com/SAP/jenkins-library/pkg/buildsettings" "github.com/SAP/jenkins-library/pkg/certutils" "github.com/SAP/jenkins-library/pkg/cnbutils" "github.com/SAP/jenkins-library/pkg/cnbutils/bindings" - "github.com/SAP/jenkins-library/pkg/cnbutils/privacy" "github.com/SAP/jenkins-library/pkg/cnbutils/project" "github.com/SAP/jenkins-library/pkg/cnbutils/project/metadata" "github.com/SAP/jenkins-library/pkg/command" @@ -34,58 +33,12 @@ const ( platformPath = "/tmp/platform" ) -type pathEnum string - -const ( - pathEnumRoot = pathEnum("root") - pathEnumFolder = pathEnum("folder") - pathEnumArchive = pathEnum("archive") -) - type cnbBuildUtilsBundle struct { *command.Command *piperutils.Files *docker.Client } -type cnbBuildTelemetry struct { - dockerImage string - Version int `json:"version"` - Data []cnbBuildTelemetryData `json:"data"` -} - -type cnbBuildTelemetryData struct { - ImageTag string `json:"imageTag"` - AdditionalTags []string `json:"additionalTags"` - BindingKeys []string `json:"bindingKeys"` - Path pathEnum `json:"path"` - BuildEnv cnbBuildTelemetryDataBuildEnv `json:"buildEnv"` - Buildpacks cnbBuildTelemetryDataBuildpacks `json:"buildpacks"` - ProjectDescriptor cnbBuildTelemetryDataProjectDescriptor `json:"projectDescriptor"` - BuildTool string `json:"buildTool"` - Builder string `json:"builder"` -} - -type cnbBuildTelemetryDataBuildEnv struct { - KeysFromConfig []string `json:"keysFromConfig"` - KeysFromProjectDescriptor []string `json:"keysFromProjectDescriptor"` - KeysOverall []string `json:"keysOverall"` - JVMVersion string `json:"jvmVersion"` - KeyValues map[string]interface{} `json:"keyValues"` -} - -type cnbBuildTelemetryDataBuildpacks struct { - FromConfig []string `json:"FromConfig"` - FromProjectDescriptor []string `json:"FromProjectDescriptor"` - Overall []string `json:"overall"` -} - -type cnbBuildTelemetryDataProjectDescriptor struct { - Used bool `json:"used"` - IncludeUsed bool `json:"includeUsed"` - ExcludeUsed bool `json:"excludeUsed"` -} - func processConfigs(main cnbBuildOptions, multipleImages []map[string]interface{}) ([]cnbBuildOptions, error) { var result []cnbBuildOptions @@ -290,7 +243,7 @@ func (config *cnbBuildOptions) mergeEnvVars(vars map[string]interface{}) { } } -func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (pathEnum, string, error) { +func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (buildpacks.PathEnum, string, error) { pwd, err := utils.Getwd() if err != nil { log.SetErrorCategory(log.ErrorBuild) @@ -298,7 +251,7 @@ func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (pathEnum, } if config.Path == "" { - return pathEnumRoot, pwd, nil + return buildpacks.PathEnumRoot, pwd, nil } matches, err := utils.Glob(config.Path) if err != nil { @@ -323,68 +276,22 @@ func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (pathEnum, } if dir { - return pathEnumFolder, source, nil + return buildpacks.PathEnumFolder, source, nil } else { - return pathEnumArchive, source, nil + return buildpacks.PathEnumArchive, source, nil } } -func addConfigTelemetryData(utils cnbutils.BuildUtils, data *cnbBuildTelemetryData, dockerImage string, config *cnbBuildOptions) { - var bindingKeys []string - for k := range config.Bindings { - bindingKeys = append(bindingKeys, k) - } - data.ImageTag = config.ContainerImageTag - data.AdditionalTags = config.AdditionalTags - data.BindingKeys = bindingKeys - data.Path, _, _ = config.resolvePath(utils) // ignore error here, telemetry problems should not fail the build - - configKeys := data.BuildEnv.KeysFromConfig - overallKeys := data.BuildEnv.KeysOverall - for key := range config.BuildEnvVars { - configKeys = append(configKeys, key) - overallKeys = append(overallKeys, key) - } - data.BuildEnv.KeysFromConfig = configKeys - data.BuildEnv.KeysOverall = overallKeys - - buildTool, _ := getBuildToolFromStageConfig("cnbBuild") // ignore error here, telemetry problems should not fail the build - data.BuildTool = buildTool - - data.Buildpacks.FromConfig = privacy.FilterBuildpacks(config.Buildpacks) - - data.Builder = privacy.FilterBuilder(dockerImage) -} - -func addProjectDescriptorTelemetryData(data *cnbBuildTelemetryData, descriptor project.Descriptor) { - descriptorKeys := data.BuildEnv.KeysFromProjectDescriptor - overallKeys := data.BuildEnv.KeysOverall - for key := range descriptor.EnvVars { - descriptorKeys = append(descriptorKeys, key) - overallKeys = append(overallKeys, key) - } - data.BuildEnv.KeysFromProjectDescriptor = descriptorKeys - data.BuildEnv.KeysOverall = overallKeys - - data.Buildpacks.FromProjectDescriptor = privacy.FilterBuildpacks(descriptor.Buildpacks) - - data.ProjectDescriptor.Used = true - data.ProjectDescriptor.IncludeUsed = descriptor.Include != nil - data.ProjectDescriptor.ExcludeUsed = descriptor.Exclude != nil -} - func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { stepName := "cnbBuild" - cnbTelemetry := &cnbBuildTelemetry{ - Version: 3, - } + telemetry := buildpacks.NewTelemetry(telemetryData) dockerImage, err := GetDockerImageValue(stepName) if err != nil { log.Entry().Warnf("failed to retrieve dockerImage configuration: '%v'", err) } - cnbTelemetry.dockerImage = dockerImage + telemetry.WithImage(dockerImage) cnbBuildConfig := buildsettings.BuildOptions{ CreateBOM: config.CreateBOM, @@ -419,7 +326,7 @@ func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, } for _, c := range mergedConfigs { - err = runCnbBuild(&c, cnbTelemetry, utils, commonPipelineEnvironment, httpClient) + err = runCnbBuild(&c, telemetry, utils, commonPipelineEnvironment, httpClient) if err != nil { return err } @@ -433,16 +340,10 @@ func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, } } - telemetryData.Custom1Label = "cnbBuildStepData" - customData, err := json.Marshal(cnbTelemetry) - if err != nil { - return errors.Wrap(err, "failed to marshal custom telemetry data") - } - telemetryData.Custom1 = string(customData) - return nil + return telemetry.Export() } -func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { +func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { err := cleanDir("/layers", utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) @@ -465,8 +366,7 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils } config.BuildEnvVars["TMPDIR"] = tempdir - customTelemetryData := cnbBuildTelemetryData{} - addConfigTelemetryData(utils, &customTelemetryData, cnbTelemetry.dockerImage, config) + telemetrySegment := createInitialTelemetrySegment(config, utils) err = isBuilder(utils) if err != nil { @@ -490,7 +390,7 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrapf(err, "failed to parse %s", projDescPath) } - addProjectDescriptorTelemetryData(&customTelemetryData, *descriptor) + telemetrySegment.WithProjectDescriptor(descriptor) config.mergeEnvVars(descriptor.EnvVars) @@ -522,9 +422,8 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrap(err, "failed to retrieve target image configuration") } - customTelemetryData.Buildpacks.Overall = privacy.FilterBuildpacks(config.Buildpacks) - customTelemetryData.BuildEnv.KeyValues = privacy.FilterEnv(config.BuildEnvVars) - cnbTelemetry.Data = append(cnbTelemetry.Data, customTelemetryData) + + telemetry.AddSegment(telemetrySegment.WithBuildpacksOverall(config.Buildpacks).WithKeyValues(config.BuildEnvVars)) if commonPipelineEnvironment.container.imageNameTag == "" { commonPipelineEnvironment.container.registryURL = fmt.Sprintf("%s://%s", targetImage.ContainerRegistry.Scheme, targetImage.ContainerRegistry.Host) @@ -565,7 +464,7 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils return errors.Wrapf(err, "failed to clean up target folder %s", target) } - if pathType != pathEnumArchive { + if pathType != buildpacks.PathEnumArchive { err = cnbutils.CopyProject(source, target, include, exclude, utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) @@ -673,7 +572,7 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils commonPipelineEnvironment.container.imageDigests = append(commonPipelineEnvironment.container.imageDigests, digest) if len(config.PreserveFiles) > 0 { - if pathType != pathEnumArchive { + if pathType != buildpacks.PathEnumArchive { err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) @@ -686,3 +585,16 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils return nil } + +func createInitialTelemetrySegment(config *cnbBuildOptions, utils cnbutils.BuildUtils) *buildpacks.Segment { + telemetrySegment := buildpacks.NewSegment() + projectPath, _, _ := config.resolvePath(utils) // ignore error here, telemetry problems should not fail the build + buildTool, _ := getBuildToolFromStageConfig("cnbBuild") // ignore error here, telemetry problems should not fail the build + + return telemetrySegment.WithBindings(config.Bindings). + WithTags(config.ContainerImageTag, config.AdditionalTags). + WithPath(projectPath). + WithEnv(config.BuildEnvVars). + WithBuildTool(buildTool). + WithBuildpacksFromConfig(config.Buildpacks) +} diff --git a/cmd/cnbBuild_test.go b/cmd/cnbBuild_test.go index f5c7592a7..12e848681 100644 --- a/cmd/cnbBuild_test.go +++ b/cmd/cnbBuild_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "testing" + "github.com/SAP/jenkins-library/pkg/buildpacks" "github.com/SAP/jenkins-library/pkg/cnbutils" piperconf "github.com/SAP/jenkins-library/pkg/config" piperhttp "github.com/SAP/jenkins-library/pkg/http" @@ -162,8 +163,8 @@ func TestRunCnbBuild(t *testing.T) { utils.FilesMock.AddFile("project.toml", []byte(projectToml)) addBuilderFiles(&utils) - telemetryData := telemetry.CustomData{} - err := callCnbBuild(&config, &telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + telemetryData := &telemetry.CustomData{} + err := callCnbBuild(&config, telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) require.NoError(t, err) runner := utils.ExecMockRunner @@ -177,8 +178,8 @@ func TestRunCnbBuild(t *testing.T) { assert.Contains(t, commonPipelineEnvironment.container.imageDigests, "sha256:52eac630560210e5ae13eb10797c4246d6f02d425f32b9430ca00bde697c79ec") customDataAsString := telemetryData.Custom1 - customData := cnbBuildTelemetry{} - err = json.Unmarshal([]byte(customDataAsString), &customData) + customData := &buildpacks.BuildpacksTelemetry{} + err = json.Unmarshal([]byte(customDataAsString), customData) require.NoError(t, err) assert.Equal(t, 1, len(customData.Data)) assert.Equal(t, "root", string(customData.Data[0].Path)) @@ -601,13 +602,13 @@ uri = "some-buildpack"`)) addBuilderFiles(&utils) - telemetryData := telemetry.CustomData{} - err := callCnbBuild(&config, &telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) + telemetryData := &telemetry.CustomData{} + err := callCnbBuild(&config, telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) require.NoError(t, err) customDataAsString := telemetryData.Custom1 - customData := cnbBuildTelemetry{} - err = json.Unmarshal([]byte(customDataAsString), &customData) + customData := &buildpacks.BuildpacksTelemetry{} + err = json.Unmarshal([]byte(customDataAsString), customData) require.NoError(t, err) assert.Equal(t, 3, customData.Version) @@ -718,13 +719,13 @@ uri = "some-buildpack" addBuilderFiles(&utils) - telemetryData := telemetry.CustomData{} - err := callCnbBuild(&config, &telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) + telemetryData := &telemetry.CustomData{} + err := callCnbBuild(&config, telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) require.NoError(t, err) customDataAsString := telemetryData.Custom1 - customData := cnbBuildTelemetry{} - err = json.Unmarshal([]byte(customDataAsString), &customData) + customData := &buildpacks.BuildpacksTelemetry{} + err = json.Unmarshal([]byte(customDataAsString), customData) require.NoError(t, err) require.Equal(t, 1, len(customData.Data)) @@ -760,13 +761,13 @@ uri = "some-buildpack" utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`)) addBuilderFiles(&utils) - telemetryData := telemetry.CustomData{} - err := callCnbBuild(&config, &telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) + telemetryData := &telemetry.CustomData{} + err := callCnbBuild(&config, telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) require.NoError(t, err) customDataAsString := telemetryData.Custom1 - customData := cnbBuildTelemetry{} - err = json.Unmarshal([]byte(customDataAsString), &customData) + customData := &buildpacks.BuildpacksTelemetry{} + err = json.Unmarshal([]byte(customDataAsString), customData) assert.NoError(t, err) require.Equal(t, expectedImageCount, len(customData.Data)) diff --git a/pkg/buildpacks/buildpacks.go b/pkg/buildpacks/buildpacks.go new file mode 100644 index 000000000..23c3ef1da --- /dev/null +++ b/pkg/buildpacks/buildpacks.go @@ -0,0 +1,9 @@ +package buildpacks + +type PathEnum string + +const ( + PathEnumRoot = PathEnum("root") + PathEnumFolder = PathEnum("folder") + PathEnumArchive = PathEnum("archive") +) diff --git a/pkg/buildpacks/telemetry.go b/pkg/buildpacks/telemetry.go new file mode 100644 index 000000000..13fe13721 --- /dev/null +++ b/pkg/buildpacks/telemetry.go @@ -0,0 +1,164 @@ +package buildpacks + +import ( + "encoding/json" + + "github.com/SAP/jenkins-library/pkg/cnbutils/privacy" + "github.com/SAP/jenkins-library/pkg/cnbutils/project" + "github.com/SAP/jenkins-library/pkg/telemetry" + "github.com/pkg/errors" +) + +const version = 3 + +type Telemetry struct { + customData *telemetry.CustomData + data *BuildpacksTelemetry +} + +func NewTelemetry(customData *telemetry.CustomData) *Telemetry { + return &Telemetry{ + customData: customData, + data: &BuildpacksTelemetry{ + Version: version, + }, + } +} + +func (d *Telemetry) Export() error { + d.customData.Custom1Label = "cnbBuildStepData" + customData, err := json.Marshal(d.data) + if err != nil { + return errors.Wrap(err, "failed to marshal custom telemetry data") + } + d.customData.Custom1 = string(customData) + return nil +} + +func (d *Telemetry) WithImage(image string) { + d.data.builder = image +} + +func (d *Telemetry) AddSegment(segment *Segment) { + segment.data.Builder = d.data.builder + d.data.Data = append(d.data.Data, segment.data) +} + +type BuildpacksTelemetry struct { + builder string + Version int `json:"version"` + Data []*cnbBuildTelemetryData `json:"data"` +} + +type cnbBuildTelemetryData struct { + ImageTag string `json:"imageTag"` + AdditionalTags []string `json:"additionalTags"` + BindingKeys []string `json:"bindingKeys"` + Path PathEnum `json:"path"` + BuildEnv cnbBuildTelemetryDataBuildEnv `json:"buildEnv"` + Buildpacks cnbBuildTelemetryDataBuildpacks `json:"buildpacks"` + ProjectDescriptor cnbBuildTelemetryDataProjectDescriptor `json:"projectDescriptor"` + BuildTool string `json:"buildTool"` + Builder string `json:"builder"` +} + +type cnbBuildTelemetryDataBuildEnv struct { + KeysFromConfig []string `json:"keysFromConfig"` + KeysFromProjectDescriptor []string `json:"keysFromProjectDescriptor"` + KeysOverall []string `json:"keysOverall"` + JVMVersion string `json:"jvmVersion"` + KeyValues map[string]interface{} `json:"keyValues"` +} + +type cnbBuildTelemetryDataBuildpacks struct { + FromConfig []string `json:"FromConfig"` + FromProjectDescriptor []string `json:"FromProjectDescriptor"` + Overall []string `json:"overall"` +} + +type cnbBuildTelemetryDataProjectDescriptor struct { + Used bool `json:"used"` + IncludeUsed bool `json:"includeUsed"` + ExcludeUsed bool `json:"excludeUsed"` +} + +type Segment struct { + data *cnbBuildTelemetryData +} + +func NewSegment() *Segment { + return &Segment{ + data: &cnbBuildTelemetryData{}, + } +} + +func (s *Segment) WithBindings(bindings map[string]interface{}) *Segment { + var bindingKeys []string + for k := range bindings { + bindingKeys = append(bindingKeys, k) + } + s.data.BindingKeys = bindingKeys + return s +} + +func (s *Segment) WithEnv(env map[string]interface{}) *Segment { + s.data.BuildEnv.KeysFromConfig = []string{} + s.data.BuildEnv.KeysOverall = []string{} + for key := range env { + s.data.BuildEnv.KeysFromConfig = append(s.data.BuildEnv.KeysFromConfig, key) + s.data.BuildEnv.KeysOverall = append(s.data.BuildEnv.KeysOverall, key) + } + return s +} + +func (s *Segment) WithTags(tag string, additionalTags []string) *Segment { + s.data.ImageTag = tag + s.data.AdditionalTags = additionalTags + return s +} + +func (s *Segment) WithPath(path PathEnum) *Segment { + s.data.Path = path + return s +} + +func (s *Segment) WithBuildTool(buildTool string) *Segment { + s.data.BuildTool = buildTool + return s +} + +func (s *Segment) WithBuilder(builder string) *Segment { + s.data.Builder = privacy.FilterBuilder(builder) + return s +} + +func (s *Segment) WithBuildpacksFromConfig(buildpacks []string) *Segment { + s.data.Buildpacks.FromConfig = privacy.FilterBuildpacks(buildpacks) + return s +} + +func (s *Segment) WithBuildpacksOverall(buildpacks []string) *Segment { + s.data.Buildpacks.Overall = privacy.FilterBuildpacks(buildpacks) + return s +} + +func (s *Segment) WithKeyValues(env map[string]interface{}) *Segment { + s.data.BuildEnv.KeyValues = privacy.FilterEnv(env) + return s +} + +func (s *Segment) WithProjectDescriptor(descriptor *project.Descriptor) *Segment { + descriptorKeys := s.data.BuildEnv.KeysFromProjectDescriptor + overallKeys := s.data.BuildEnv.KeysOverall + for key := range descriptor.EnvVars { + descriptorKeys = append(descriptorKeys, key) + overallKeys = append(overallKeys, key) + } + s.data.BuildEnv.KeysFromProjectDescriptor = descriptorKeys + s.data.BuildEnv.KeysOverall = overallKeys + s.data.Buildpacks.FromProjectDescriptor = privacy.FilterBuildpacks(descriptor.Buildpacks) + s.data.ProjectDescriptor.Used = true + s.data.ProjectDescriptor.IncludeUsed = descriptor.Include != nil + s.data.ProjectDescriptor.ExcludeUsed = descriptor.Exclude != nil + return s +}