1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2024-12-12 10:55:20 +02:00

feat(cnbBuild): added dockerimage to the telemetry data (#3501)

Co-authored-by: Philipp Stehle <philipp.stehle@sap.com>
Co-authored-by: Sumit Kulhadia <sumit.kulhadia@sap.com>
Co-authored-by: Johannes Dillmann <j.dillmann@sap.com>
This commit is contained in:
Ralf Pannemans 2022-02-16 13:28:51 +01:00 committed by GitHub
parent 4b2f61589d
commit e7db09db12
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 85 additions and 43 deletions

View File

@ -60,6 +60,7 @@ type cnbBuildTelemetryData struct {
Buildpacks cnbBuildTelemetryDataBuildpacks `json:"buildpacks"`
ProjectDescriptor cnbBuildTelemetryDataProjectDescriptor `json:"projectDescriptor"`
BuildTool string `json:"buildTool"`
Builder string `json:"builder"`
}
type cnbBuildTelemetryDataBuildEnv struct {
@ -139,11 +140,8 @@ func cnbBuild(config cnbBuildOptions, telemetryData *telemetry.CustomData, commo
utils := newCnbBuildUtils()
client := &piperhttp.Client{}
cnbTelemetry := cnbBuildTelemetry{
Version: 2,
}
err := callCnbBuild(&config, telemetryData, &cnbTelemetry, utils, commonPipelineEnvironment, client)
err := callCnbBuild(&config, telemetryData, utils, commonPipelineEnvironment, client)
if err != nil {
log.Entry().WithError(err).Fatal("step execution failed")
}
@ -402,6 +400,14 @@ func addConfigTelemetryData(utils cnbutils.BuildUtils, data *cnbBuildTelemetryDa
data.BuildTool = buildTool
data.Buildpacks.FromConfig = privacy.FilterBuildpacks(config.Buildpacks)
dockerImage, err := getDockerImageValue("cnbBuild")
if err != nil {
log.Entry().Warnf("Error while preparing telemetry: retrieving docker image failed: '%v'", err)
data.Builder = ""
} else {
data.Builder = privacy.FilterBuilder(dockerImage)
}
}
func addProjectDescriptorTelemetryData(data *cnbBuildTelemetryData, descriptor project.Descriptor) {
@ -421,7 +427,10 @@ func addProjectDescriptorTelemetryData(data *cnbBuildTelemetryData, descriptor p
data.ProjectDescriptor.ExcludeUsed = descriptor.Exclude != nil
}
func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, telemetry *cnbBuildTelemetry, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error {
func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error {
telemetry := &cnbBuildTelemetry{
Version: 3,
}
mergedConfigs, err := processConfigs(*config, config.MultipleImages)
if err != nil {
return errors.Wrap(err, "failed to process config")

View File

@ -7,6 +7,7 @@ import (
"testing"
"github.com/SAP/jenkins-library/pkg/cnbutils"
piperconf "github.com/SAP/jenkins-library/pkg/config"
piperhttp "github.com/SAP/jenkins-library/pkg/http"
"github.com/SAP/jenkins-library/pkg/mock"
"github.com/SAP/jenkins-library/pkg/telemetry"
@ -38,7 +39,7 @@ func assertLifecycleCalls(t *testing.T, runner *mock.ExecMockRunner, callNo int)
}
func TestRunCnbBuild(t *testing.T) {
t.Parallel()
configOptions.openFile = piperconf.OpenPiperFile
t.Run("prefers direct configuration", func(t *testing.T) {
t.Parallel()
@ -59,7 +60,7 @@ func TestRunCnbBuild(t *testing.T) {
utils.FilesMock.AddFile("project.toml", []byte(projectToml))
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
require.NoError(t, err)
runner := utils.ExecMockRunner
@ -90,7 +91,7 @@ func TestRunCnbBuild(t *testing.T) {
addBuilderFiles(&utils)
telemetryData := telemetry.CustomData{}
err := callCnbBuild(&config, &telemetryData, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
require.NoError(t, err)
runner := utils.ExecMockRunner
@ -122,7 +123,7 @@ func TestRunCnbBuild(t *testing.T) {
utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
require.NoError(t, err)
runner := utils.ExecMockRunner
@ -147,7 +148,7 @@ func TestRunCnbBuild(t *testing.T) {
utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
require.NoError(t, err)
runner := utils.ExecMockRunner
@ -177,7 +178,7 @@ func TestRunCnbBuild(t *testing.T) {
utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
require.NoError(t, err)
runner := utils.ExecMockRunner
@ -220,7 +221,7 @@ func TestRunCnbBuild(t *testing.T) {
utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, client)
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, client)
require.NoError(t, err)
result, err := utils.FilesMock.FileRead(caCertsTmpFile)
@ -250,7 +251,7 @@ func TestRunCnbBuild(t *testing.T) {
utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
require.NoError(t, err)
runner := utils.ExecMockRunner
@ -277,7 +278,7 @@ func TestRunCnbBuild(t *testing.T) {
utils.FilesMock.AddFile("/workspace/pom.xml", []byte("test"))
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{})
require.NoError(t, err)
runner := utils.ExecMockRunner
@ -301,7 +302,7 @@ func TestRunCnbBuild(t *testing.T) {
utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{})
require.NoError(t, err)
runner := utils.ExecMockRunner
@ -324,7 +325,7 @@ func TestRunCnbBuild(t *testing.T) {
utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":"dXNlcjpwYXNz"}}`))
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
assert.EqualError(t, err, "failed to generate CNB_REGISTRY_AUTH: json: cannot unmarshal string into Go struct field ConfigFile.auths of type types.AuthConfig")
})
@ -341,7 +342,7 @@ func TestRunCnbBuild(t *testing.T) {
utils := newCnbBuildTestsUtils()
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
assert.EqualError(t, err, "failed to generate CNB_REGISTRY_AUTH: could not read 'not-there/config.json'")
})
@ -358,7 +359,7 @@ func TestRunCnbBuild(t *testing.T) {
utils := newCnbBuildTestsUtils()
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
assert.EqualError(t, err, "failed to rename DockerConfigJSON file 'not-there': renaming file 'not-there' is not supported, since it does not exist, or is not a leaf-entry")
})
@ -369,7 +370,7 @@ func TestRunCnbBuild(t *testing.T) {
utils := newCnbBuildTestsUtils()
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
assert.EqualError(t, err, "the provided dockerImage is not a valid builder: binary '/cnb/lifecycle/creator' not found")
})
@ -389,7 +390,7 @@ func TestRunCnbBuild(t *testing.T) {
utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
addBuilderFiles(&utils)
err := callCnbBuild(&config, &telemetry.CustomData{}, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
assert.EqualError(t, err, "failed to copy certificates: cannot copy '/etc/ssl/certs/ca-certificates.crt': file does not exist")
})
@ -429,8 +430,7 @@ uri = "some-buildpack"`))
addBuilderFiles(&utils)
telemetryData := telemetry.CustomData{}
cnbTelemetry := cnbBuildTelemetry{Version: 2}
err := callCnbBuild(&config, &telemetryData, &cnbTelemetry, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
require.NoError(t, err)
customDataAsString := telemetryData.Custom1
@ -438,12 +438,13 @@ uri = "some-buildpack"`))
err = json.Unmarshal([]byte(customDataAsString), &customData)
require.NoError(t, err)
assert.Equal(t, 2, customData.Version)
assert.Equal(t, 3, customData.Version)
require.Equal(t, 1, len(customData.Data))
assert.Equal(t, "3.1.5", customData.Data[0].ImageTag)
assert.Equal(t, "folder", string(customData.Data[0].Path))
assert.Contains(t, customData.Data[0].AdditionalTags, "latest")
assert.Contains(t, customData.Data[0].BindingKeys, "SECRET")
assert.Equal(t, "paketobuildpacks/builder:full", customData.Data[0].Builder)
assert.Contains(t, customData.Data[0].Buildpacks.FromConfig, "paketobuildpacks/java")
assert.NotContains(t, customData.Data[0].Buildpacks.FromProjectDescriptor, "paketobuildpacks/java")
@ -490,8 +491,7 @@ uri = "some-buildpack"
addBuilderFiles(&utils)
telemetryData := telemetry.CustomData{}
cnbTelemetry := cnbBuildTelemetry{Version: 2}
err := callCnbBuild(&config, &telemetryData, &cnbTelemetry, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
require.NoError(t, err)
customDataAsString := telemetryData.Custom1
@ -533,7 +533,7 @@ uri = "some-buildpack"
addBuilderFiles(&utils)
telemetryData := telemetry.CustomData{}
err := callCnbBuild(&config, &telemetryData, &cnbBuildTelemetry{}, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
err := callCnbBuild(&config, &telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{})
require.NoError(t, err)
customDataAsString := telemetryData.Custom1

View File

@ -17,12 +17,12 @@ import (
)
type configCommandOptions struct {
output string //output format, so far only JSON
outputFile string //if set: path to file where the output should be written to
parametersJSON string //parameters to be considered in JSON format
output string // output format, so far only JSON
outputFile string // if set: path to file where the output should be written to
parametersJSON string // parameters to be considered in JSON format
stageConfig bool
stageConfigAcceptedParameters []string
stepMetadata string //metadata to be considered, can be filePath or ENV containing JSON in format 'ENV:MY_ENV_VAR'
stepMetadata string // metadata to be considered, can be filePath or ENV containing JSON in format 'ENV:MY_ENV_VAR'
stepName string
contextConfig bool
openFile func(s string, t map[string]string) (io.ReadCloser, error)
@ -115,7 +115,7 @@ func getStageConfig() (config.StepConfig, error) {
customConfig, err := configOptions.openFile(projectConfigFile, GeneralConfig.GitHubAccessTokens)
if err != nil {
if !os.IsNotExist(err) {
if !errors.Is(err, os.ErrNotExist) {
return stepConfig, errors.Wrapf(err, "config: open configuration file '%v' failed", projectConfigFile)
}
customConfig = nil
@ -171,7 +171,7 @@ func getConfig() (config.StepConfig, error) {
customConfig, err := configOptions.openFile(projectConfigFile, GeneralConfig.GitHubAccessTokens)
if err != nil {
if !os.IsNotExist(err) {
if !errors.Is(err, os.ErrNotExist) {
return stepConfig, errors.Wrapf(err, "config: open configuration file '%v' failed", projectConfigFile)
}
customConfig = nil
@ -235,7 +235,7 @@ func generateConfig(utils getConfigUtils) error {
func addConfigFlags(cmd *cobra.Command) {
//ToDo: support more output options, like https://kubernetes.io/docs/reference/kubectl/overview/#formatting-output
// ToDo: support more output options, like https://kubernetes.io/docs/reference/kubectl/overview/#formatting-output
cmd.Flags().StringVar(&configOptions.output, "output", "json", "Defines the output format")
cmd.Flags().StringVar(&configOptions.outputFile, "outputFile", "", "Defines a file path. f set, the output will be written to the defines file")
@ -256,20 +256,20 @@ func defaultsAndFilters(metadata *config.StepData, stepName string) ([]io.ReadCl
}
return []io.ReadCloser{defaults}, metadata.GetContextParameterFilters(), nil
}
//ToDo: retrieve default values from metadata
// ToDo: retrieve default values from metadata
return []io.ReadCloser{}, metadata.GetParameterFilters(), nil
}
func applyContextConditions(metadata config.StepData, stepConfig *config.StepConfig) {
//consider conditions for context configuration
// consider conditions for context configuration
//containers
// containers
config.ApplyContainerConditions(metadata.Spec.Containers, stepConfig)
//sidecars
// sidecars
config.ApplyContainerConditions(metadata.Spec.Sidecars, stepConfig)
//ToDo: remove all unnecessary sub maps?
// ToDo: remove all unnecessary sub maps?
// e.g. extract delete() from applyContainerConditions - loop over all stepConfig.Config[param.Value] and remove ...
}
@ -283,7 +283,7 @@ func prepareOutputEnvironment(outputResources []config.StepResources, envRootPat
paramPath = path.Join(paramPath, paramFields[0]["name"])
}
}
if _, err := os.Stat(filepath.Dir(paramPath)); os.IsNotExist(err) {
if _, err := os.Stat(filepath.Dir(paramPath)); errors.Is(err, os.ErrNotExist) {
log.Entry().Debugf("Creating directory: %v", filepath.Dir(paramPath))
os.MkdirAll(filepath.Dir(paramPath), 0777)
}
@ -299,7 +299,7 @@ func prepareOutputEnvironment(outputResources []config.StepResources, envRootPat
}
for _, dir := range stepOutputDirectories {
if _, err := os.Stat(dir); os.IsNotExist(err) {
if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) {
log.Entry().Debugf("Creating directory: %v", dir)
os.MkdirAll(dir, 0777)
}

View File

@ -29,6 +29,11 @@ var allowedBuildpackSources = []struct {
},
}
func FilterBuilder(builder string) string {
result := FilterBuildpacks([]string{builder})
return result[0]
}
// FilterBuildpacks filters a list of buildpacks to redact Personally Identifiable Information (PII) like the hostname of a personal registry
func FilterBuildpacks(buildpacks []string) []string {
result := make([]string, 0, len(buildpacks))

View File

@ -110,3 +110,30 @@ func TestCnbPrivacy_FilterEnv(t *testing.T) {
assert.Empty(t, filteredEnv)
})
}
func TestCnbPrivacy_FilterBuilder(t *testing.T) {
t.Parallel()
t.Run("allows paketo", func(t *testing.T) {
builder := []string{
"paketobuildpacks/builder:tiny",
"paketobuildpacks/builder:base",
"paketobuildpacks/builder:full",
}
for _, b := range builder {
filteredBuilder := privacy.FilterBuilder(b)
assert.Equal(t, b, filteredBuilder)
}
})
t.Run("filters unknown builders", func(t *testing.T) {
builder := "notpaketobuildpacks/builder:base"
filteredBuilder := privacy.FilterBuilder(builder)
assert.Equal(t, "<redacted>", filteredBuilder)
})
}

View File

@ -222,7 +222,7 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri
if err != nil {
log.Entry().Warnf("failed to parse parameters from environment: %v", err)
} else {
//apply aliases
// apply aliases
for _, p := range parameters {
params = setParamValueFromAlias(stepName, params, filters.Parameters, p.Name, p.Aliases)
}
@ -293,7 +293,7 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri
}
// SetVaultCredentials sets the appRoleID and the appRoleSecretID or the vaultTokento load additional
//configuration from vault
// configuration from vault
// Either appRoleID and appRoleSecretID or vaultToken must be specified.
func (c *Config) SetVaultCredentials(appRoleID, appRoleSecretID string, vaultToken string) {
c.vaultCredentials = VaultCredentials{
@ -359,8 +359,9 @@ func GetYAML(data interface{}) (string, error) {
// OpenPiperFile provides functionality to retrieve configuration via file or http
func OpenPiperFile(name string, accessTokens map[string]string) (io.ReadCloser, error) {
if len(name) == 0 {
return nil, fmt.Errorf("no filename provided")
return nil, errors.Wrap(os.ErrNotExist, "no filename provided")
}
if !strings.HasPrefix(name, "http://") && !strings.HasPrefix(name, "https://") {
return os.Open(name)
}