Refactor buildpacks telemetry (#4467)

cmd/cnbBuild.go

@@ -2,17 +2,16 @@ package cmd
 
 import (
 	"archive/zip"
-	"encoding/json"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
 
+	"github.com/SAP/jenkins-library/pkg/buildpacks"
 	"github.com/SAP/jenkins-library/pkg/buildsettings"
 	"github.com/SAP/jenkins-library/pkg/certutils"
 	"github.com/SAP/jenkins-library/pkg/cnbutils"
 	"github.com/SAP/jenkins-library/pkg/cnbutils/bindings"
-	"github.com/SAP/jenkins-library/pkg/cnbutils/privacy"
 	"github.com/SAP/jenkins-library/pkg/cnbutils/project"
 	"github.com/SAP/jenkins-library/pkg/cnbutils/project/metadata"
 	"github.com/SAP/jenkins-library/pkg/command"
@@ -34,58 +33,12 @@ const (
 	platformPath = "/tmp/platform"
 )
 
-type pathEnum string
-
-const (
-	pathEnumRoot    = pathEnum("root")
-	pathEnumFolder  = pathEnum("folder")
-	pathEnumArchive = pathEnum("archive")
-)
-
 type cnbBuildUtilsBundle struct {
 	*command.Command
 	*piperutils.Files
 	*docker.Client
 }
 
-type cnbBuildTelemetry struct {
-	dockerImage string
-	Version     int                     `json:"version"`
-	Data        []cnbBuildTelemetryData `json:"data"`
-}
-
-type cnbBuildTelemetryData struct {
-	ImageTag          string                                 `json:"imageTag"`
-	AdditionalTags    []string                               `json:"additionalTags"`
-	BindingKeys       []string                               `json:"bindingKeys"`
-	Path              pathEnum                               `json:"path"`
-	BuildEnv          cnbBuildTelemetryDataBuildEnv          `json:"buildEnv"`
-	Buildpacks        cnbBuildTelemetryDataBuildpacks        `json:"buildpacks"`
-	ProjectDescriptor cnbBuildTelemetryDataProjectDescriptor `json:"projectDescriptor"`
-	BuildTool         string                                 `json:"buildTool"`
-	Builder           string                                 `json:"builder"`
-}
-
-type cnbBuildTelemetryDataBuildEnv struct {
-	KeysFromConfig            []string               `json:"keysFromConfig"`
-	KeysFromProjectDescriptor []string               `json:"keysFromProjectDescriptor"`
-	KeysOverall               []string               `json:"keysOverall"`
-	JVMVersion                string                 `json:"jvmVersion"`
-	KeyValues                 map[string]interface{} `json:"keyValues"`
-}
-
-type cnbBuildTelemetryDataBuildpacks struct {
-	FromConfig            []string `json:"FromConfig"`
-	FromProjectDescriptor []string `json:"FromProjectDescriptor"`
-	Overall               []string `json:"overall"`
-}
-
-type cnbBuildTelemetryDataProjectDescriptor struct {
-	Used        bool `json:"used"`
-	IncludeUsed bool `json:"includeUsed"`
-	ExcludeUsed bool `json:"excludeUsed"`
-}
-
 func processConfigs(main cnbBuildOptions, multipleImages []map[string]interface{}) ([]cnbBuildOptions, error) {
 	var result []cnbBuildOptions
 
@@ -290,7 +243,7 @@ func (config *cnbBuildOptions) mergeEnvVars(vars map[string]interface{}) {
 	}
 }
 
-func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (pathEnum, string, error) {
+func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (buildpacks.PathEnum, string, error) {
 	pwd, err := utils.Getwd()
 	if err != nil {
 		log.SetErrorCategory(log.ErrorBuild)
@@ -298,7 +251,7 @@ func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (pathEnum,
 	}
 
 	if config.Path == "" {
-		return pathEnumRoot, pwd, nil
+		return buildpacks.PathEnumRoot, pwd, nil
 	}
 	matches, err := utils.Glob(config.Path)
 	if err != nil {
@@ -323,68 +276,22 @@ func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (pathEnum,
 	}
 
 	if dir {
-		return pathEnumFolder, source, nil
+		return buildpacks.PathEnumFolder, source, nil
 	} else {
-		return pathEnumArchive, source, nil
+		return buildpacks.PathEnumArchive, source, nil
 	}
 }
 
-func addConfigTelemetryData(utils cnbutils.BuildUtils, data *cnbBuildTelemetryData, dockerImage string, config *cnbBuildOptions) {
-	var bindingKeys []string
-	for k := range config.Bindings {
-		bindingKeys = append(bindingKeys, k)
-	}
-	data.ImageTag = config.ContainerImageTag
-	data.AdditionalTags = config.AdditionalTags
-	data.BindingKeys = bindingKeys
-	data.Path, _, _ = config.resolvePath(utils) // ignore error here, telemetry problems should not fail the build
-
-	configKeys := data.BuildEnv.KeysFromConfig
-	overallKeys := data.BuildEnv.KeysOverall
-	for key := range config.BuildEnvVars {
-		configKeys = append(configKeys, key)
-		overallKeys = append(overallKeys, key)
-	}
-	data.BuildEnv.KeysFromConfig = configKeys
-	data.BuildEnv.KeysOverall = overallKeys
-
-	buildTool, _ := getBuildToolFromStageConfig("cnbBuild") // ignore error here, telemetry problems should not fail the build
-	data.BuildTool = buildTool
-
-	data.Buildpacks.FromConfig = privacy.FilterBuildpacks(config.Buildpacks)
-
-	data.Builder = privacy.FilterBuilder(dockerImage)
-}
-
-func addProjectDescriptorTelemetryData(data *cnbBuildTelemetryData, descriptor project.Descriptor) {
-	descriptorKeys := data.BuildEnv.KeysFromProjectDescriptor
-	overallKeys := data.BuildEnv.KeysOverall
-	for key := range descriptor.EnvVars {
-		descriptorKeys = append(descriptorKeys, key)
-		overallKeys = append(overallKeys, key)
-	}
-	data.BuildEnv.KeysFromProjectDescriptor = descriptorKeys
-	data.BuildEnv.KeysOverall = overallKeys
-
-	data.Buildpacks.FromProjectDescriptor = privacy.FilterBuildpacks(descriptor.Buildpacks)
-
-	data.ProjectDescriptor.Used = true
-	data.ProjectDescriptor.IncludeUsed = descriptor.Include != nil
-	data.ProjectDescriptor.ExcludeUsed = descriptor.Exclude != nil
-}
-
 func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error {
 	stepName := "cnbBuild"
-	cnbTelemetry := &cnbBuildTelemetry{
+	telemetry := buildpacks.NewTelemetry(telemetryData)
-		Version: 3,
-	}
 
 	dockerImage, err := GetDockerImageValue(stepName)
 	if err != nil {
 		log.Entry().Warnf("failed to retrieve dockerImage configuration: '%v'", err)
 	}
 
-	cnbTelemetry.dockerImage = dockerImage
+	telemetry.WithImage(dockerImage)
 
 	cnbBuildConfig := buildsettings.BuildOptions{
 		CreateBOM:         config.CreateBOM,
@@ -419,7 +326,7 @@ func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData,
 	}
 
 	for _, c := range mergedConfigs {
-		err = runCnbBuild(&c, cnbTelemetry, utils, commonPipelineEnvironment, httpClient)
+		err = runCnbBuild(&c, telemetry, utils, commonPipelineEnvironment, httpClient)
 		if err != nil {
 			return err
 		}
@@ -433,16 +340,10 @@ func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData,
 		}
 	}
 
-	telemetryData.Custom1Label = "cnbBuildStepData"
+	return telemetry.Export()
-	customData, err := json.Marshal(cnbTelemetry)
-	if err != nil {
-		return errors.Wrap(err, "failed to marshal custom telemetry data")
-	}
-	telemetryData.Custom1 = string(customData)
-	return nil
 }
 
-func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error {
+func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error {
 	err := cleanDir("/layers", utils)
 	if err != nil {
 		log.SetErrorCategory(log.ErrorBuild)
@@ -465,8 +366,7 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils
 	}
 	config.BuildEnvVars["TMPDIR"] = tempdir
 
-	customTelemetryData := cnbBuildTelemetryData{}
+	telemetrySegment := createInitialTelemetrySegment(config, utils)
-	addConfigTelemetryData(utils, &customTelemetryData, cnbTelemetry.dockerImage, config)
 
 	err = isBuilder(utils)
 	if err != nil {
@@ -490,7 +390,7 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils
 			log.SetErrorCategory(log.ErrorConfiguration)
 			return errors.Wrapf(err, "failed to parse %s", projDescPath)
 		}
-		addProjectDescriptorTelemetryData(&customTelemetryData, *descriptor)
+		telemetrySegment.WithProjectDescriptor(descriptor)
 
 		config.mergeEnvVars(descriptor.EnvVars)
 
@@ -522,9 +422,8 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils
 		log.SetErrorCategory(log.ErrorConfiguration)
 		return errors.Wrap(err, "failed to retrieve target image configuration")
 	}
-	customTelemetryData.Buildpacks.Overall = privacy.FilterBuildpacks(config.Buildpacks)
+
-	customTelemetryData.BuildEnv.KeyValues = privacy.FilterEnv(config.BuildEnvVars)
+	telemetry.AddSegment(telemetrySegment.WithBuildpacksOverall(config.Buildpacks).WithKeyValues(config.BuildEnvVars))
-	cnbTelemetry.Data = append(cnbTelemetry.Data, customTelemetryData)
 
 	if commonPipelineEnvironment.container.imageNameTag == "" {
 		commonPipelineEnvironment.container.registryURL = fmt.Sprintf("%s://%s", targetImage.ContainerRegistry.Scheme, targetImage.ContainerRegistry.Host)
@@ -565,7 +464,7 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils
 		return errors.Wrapf(err, "failed to clean up target folder %s", target)
 	}
 
-	if pathType != pathEnumArchive {
+	if pathType != buildpacks.PathEnumArchive {
 		err = cnbutils.CopyProject(source, target, include, exclude, utils)
 		if err != nil {
 			log.SetErrorCategory(log.ErrorBuild)
@@ -673,7 +572,7 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils
 	commonPipelineEnvironment.container.imageDigests = append(commonPipelineEnvironment.container.imageDigests, digest)
 
 	if len(config.PreserveFiles) > 0 {
-		if pathType != pathEnumArchive {
+		if pathType != buildpacks.PathEnumArchive {
 			err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils)
 			if err != nil {
 				log.SetErrorCategory(log.ErrorBuild)
@@ -686,3 +585,16 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils
 
 	return nil
 }
+
+func createInitialTelemetrySegment(config *cnbBuildOptions, utils cnbutils.BuildUtils) *buildpacks.Segment {
+	telemetrySegment := buildpacks.NewSegment()
+	projectPath, _, _ := config.resolvePath(utils)          // ignore error here, telemetry problems should not fail the build
+	buildTool, _ := getBuildToolFromStageConfig("cnbBuild") // ignore error here, telemetry problems should not fail the build
+
+	return telemetrySegment.WithBindings(config.Bindings).
+		WithTags(config.ContainerImageTag, config.AdditionalTags).
+		WithPath(projectPath).
+		WithEnv(config.BuildEnvVars).
+		WithBuildTool(buildTool).
+		WithBuildpacksFromConfig(config.Buildpacks)
+}

cmd/cnbBuild_test.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/SAP/jenkins-library/pkg/buildpacks"
 	"github.com/SAP/jenkins-library/pkg/cnbutils"
 	piperconf "github.com/SAP/jenkins-library/pkg/config"
 	piperhttp "github.com/SAP/jenkins-library/pkg/http"
@@ -162,8 +163,8 @@ func TestRunCnbBuild(t *testing.T) {
 		utils.FilesMock.AddFile("project.toml", []byte(projectToml))
 		addBuilderFiles(&utils)
 
-		telemetryData := telemetry.CustomData{}
+		telemetryData := &telemetry.CustomData{}
-		err := callCnbBuild(&config, &telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
+		err := callCnbBuild(&config, telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
 
 		require.NoError(t, err)
 		runner := utils.ExecMockRunner
@@ -177,8 +178,8 @@ func TestRunCnbBuild(t *testing.T) {
 		assert.Contains(t, commonPipelineEnvironment.container.imageDigests, "sha256:52eac630560210e5ae13eb10797c4246d6f02d425f32b9430ca00bde697c79ec")
 
 		customDataAsString := telemetryData.Custom1
-		customData := cnbBuildTelemetry{}
+		customData := &buildpacks.BuildpacksTelemetry{}
-		err = json.Unmarshal([]byte(customDataAsString), &customData)
+		err = json.Unmarshal([]byte(customDataAsString), customData)
 		require.NoError(t, err)
 		assert.Equal(t, 1, len(customData.Data))
 		assert.Equal(t, "root", string(customData.Data[0].Path))
@@ -601,13 +602,13 @@ uri = "some-buildpack"`))
 
 		addBuilderFiles(&utils)
 
-		telemetryData := telemetry.CustomData{}
+		telemetryData := &telemetry.CustomData{}
-		err := callCnbBuild(&config, &telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{})
+		err := callCnbBuild(&config, telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{})
 		require.NoError(t, err)
 
 		customDataAsString := telemetryData.Custom1
-		customData := cnbBuildTelemetry{}
+		customData := &buildpacks.BuildpacksTelemetry{}
-		err = json.Unmarshal([]byte(customDataAsString), &customData)
+		err = json.Unmarshal([]byte(customDataAsString), customData)
 
 		require.NoError(t, err)
 		assert.Equal(t, 3, customData.Version)
@@ -718,13 +719,13 @@ uri = "some-buildpack"
 
 		addBuilderFiles(&utils)
 
-		telemetryData := telemetry.CustomData{}
+		telemetryData := &telemetry.CustomData{}
-		err := callCnbBuild(&config, &telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{})
+		err := callCnbBuild(&config, telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{})
 		require.NoError(t, err)
 
 		customDataAsString := telemetryData.Custom1
-		customData := cnbBuildTelemetry{}
+		customData := &buildpacks.BuildpacksTelemetry{}
-		err = json.Unmarshal([]byte(customDataAsString), &customData)
+		err = json.Unmarshal([]byte(customDataAsString), customData)
 
 		require.NoError(t, err)
 		require.Equal(t, 1, len(customData.Data))
@@ -760,13 +761,13 @@ uri = "some-buildpack"
 		utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
 		addBuilderFiles(&utils)
 
-		telemetryData := telemetry.CustomData{}
+		telemetryData := &telemetry.CustomData{}
-		err := callCnbBuild(&config, &telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
+		err := callCnbBuild(&config, telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
 		require.NoError(t, err)
 
 		customDataAsString := telemetryData.Custom1
-		customData := cnbBuildTelemetry{}
+		customData := &buildpacks.BuildpacksTelemetry{}
-		err = json.Unmarshal([]byte(customDataAsString), &customData)
+		err = json.Unmarshal([]byte(customDataAsString), customData)
 		assert.NoError(t, err)
 		require.Equal(t, expectedImageCount, len(customData.Data))
 

pkg/buildpacks/buildpacks.go

@@ -0,0 +1,9 @@
+package buildpacks
+
+type PathEnum string
+
+const (
+	PathEnumRoot    = PathEnum("root")
+	PathEnumFolder  = PathEnum("folder")
+	PathEnumArchive = PathEnum("archive")
+)

pkg/buildpacks/telemetry.go

@@ -0,0 +1,164 @@
+package buildpacks
+
+import (
+	"encoding/json"
+
+	"github.com/SAP/jenkins-library/pkg/cnbutils/privacy"
+	"github.com/SAP/jenkins-library/pkg/cnbutils/project"
+	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/pkg/errors"
+)
+
+const version = 3
+
+type Telemetry struct {
+	customData *telemetry.CustomData
+	data       *BuildpacksTelemetry
+}
+
+func NewTelemetry(customData *telemetry.CustomData) *Telemetry {
+	return &Telemetry{
+		customData: customData,
+		data: &BuildpacksTelemetry{
+			Version: version,
+		},
+	}
+}
+
+func (d *Telemetry) Export() error {
+	d.customData.Custom1Label = "cnbBuildStepData"
+	customData, err := json.Marshal(d.data)
+	if err != nil {
+		return errors.Wrap(err, "failed to marshal custom telemetry data")
+	}
+	d.customData.Custom1 = string(customData)
+	return nil
+}
+
+func (d *Telemetry) WithImage(image string) {
+	d.data.builder = image
+}
+
+func (d *Telemetry) AddSegment(segment *Segment) {
+	segment.data.Builder = d.data.builder
+	d.data.Data = append(d.data.Data, segment.data)
+}
+
+type BuildpacksTelemetry struct {
+	builder string
+	Version int                      `json:"version"`
+	Data    []*cnbBuildTelemetryData `json:"data"`
+}
+
+type cnbBuildTelemetryData struct {
+	ImageTag          string                                 `json:"imageTag"`
+	AdditionalTags    []string                               `json:"additionalTags"`
+	BindingKeys       []string                               `json:"bindingKeys"`
+	Path              PathEnum                               `json:"path"`
+	BuildEnv          cnbBuildTelemetryDataBuildEnv          `json:"buildEnv"`
+	Buildpacks        cnbBuildTelemetryDataBuildpacks        `json:"buildpacks"`
+	ProjectDescriptor cnbBuildTelemetryDataProjectDescriptor `json:"projectDescriptor"`
+	BuildTool         string                                 `json:"buildTool"`
+	Builder           string                                 `json:"builder"`
+}
+
+type cnbBuildTelemetryDataBuildEnv struct {
+	KeysFromConfig            []string               `json:"keysFromConfig"`
+	KeysFromProjectDescriptor []string               `json:"keysFromProjectDescriptor"`
+	KeysOverall               []string               `json:"keysOverall"`
+	JVMVersion                string                 `json:"jvmVersion"`
+	KeyValues                 map[string]interface{} `json:"keyValues"`
+}
+
+type cnbBuildTelemetryDataBuildpacks struct {
+	FromConfig            []string `json:"FromConfig"`
+	FromProjectDescriptor []string `json:"FromProjectDescriptor"`
+	Overall               []string `json:"overall"`
+}
+
+type cnbBuildTelemetryDataProjectDescriptor struct {
+	Used        bool `json:"used"`
+	IncludeUsed bool `json:"includeUsed"`
+	ExcludeUsed bool `json:"excludeUsed"`
+}
+
+type Segment struct {
+	data *cnbBuildTelemetryData
+}
+
+func NewSegment() *Segment {
+	return &Segment{
+		data: &cnbBuildTelemetryData{},
+	}
+}
+
+func (s *Segment) WithBindings(bindings map[string]interface{}) *Segment {
+	var bindingKeys []string
+	for k := range bindings {
+		bindingKeys = append(bindingKeys, k)
+	}
+	s.data.BindingKeys = bindingKeys
+	return s
+}
+
+func (s *Segment) WithEnv(env map[string]interface{}) *Segment {
+	s.data.BuildEnv.KeysFromConfig = []string{}
+	s.data.BuildEnv.KeysOverall = []string{}
+	for key := range env {
+		s.data.BuildEnv.KeysFromConfig = append(s.data.BuildEnv.KeysFromConfig, key)
+		s.data.BuildEnv.KeysOverall = append(s.data.BuildEnv.KeysOverall, key)
+	}
+	return s
+}
+
+func (s *Segment) WithTags(tag string, additionalTags []string) *Segment {
+	s.data.ImageTag = tag
+	s.data.AdditionalTags = additionalTags
+	return s
+}
+
+func (s *Segment) WithPath(path PathEnum) *Segment {
+	s.data.Path = path
+	return s
+}
+
+func (s *Segment) WithBuildTool(buildTool string) *Segment {
+	s.data.BuildTool = buildTool
+	return s
+}
+
+func (s *Segment) WithBuilder(builder string) *Segment {
+	s.data.Builder = privacy.FilterBuilder(builder)
+	return s
+}
+
+func (s *Segment) WithBuildpacksFromConfig(buildpacks []string) *Segment {
+	s.data.Buildpacks.FromConfig = privacy.FilterBuildpacks(buildpacks)
+	return s
+}
+
+func (s *Segment) WithBuildpacksOverall(buildpacks []string) *Segment {
+	s.data.Buildpacks.Overall = privacy.FilterBuildpacks(buildpacks)
+	return s
+}
+
+func (s *Segment) WithKeyValues(env map[string]interface{}) *Segment {
+	s.data.BuildEnv.KeyValues = privacy.FilterEnv(env)
+	return s
+}
+
+func (s *Segment) WithProjectDescriptor(descriptor *project.Descriptor) *Segment {
+	descriptorKeys := s.data.BuildEnv.KeysFromProjectDescriptor
+	overallKeys := s.data.BuildEnv.KeysOverall
+	for key := range descriptor.EnvVars {
+		descriptorKeys = append(descriptorKeys, key)
+		overallKeys = append(overallKeys, key)
+	}
+	s.data.BuildEnv.KeysFromProjectDescriptor = descriptorKeys
+	s.data.BuildEnv.KeysOverall = overallKeys
+	s.data.Buildpacks.FromProjectDescriptor = privacy.FilterBuildpacks(descriptor.Buildpacks)
+	s.data.ProjectDescriptor.Used = true
+	s.data.ProjectDescriptor.IncludeUsed = descriptor.Include != nil
+	s.data.ProjectDescriptor.ExcludeUsed = descriptor.Exclude != nil
+	return s
+}