jenkins/sap-jenkins-library
1
0
Fork 0
mirror of https://github.com/SAP/jenkins-library.git synced 2024-12-14 11:03:09 +02:00
Code Issues Releases Activity

Merge pull request #945 from SAP/cfg-hierarchical-defaults

Browse Source 
Add capability for hierarchical defaults in golang
This commit is contained in:
Sven Merk 2019-11-06 12:20:33 +01:00 committed by GitHub
commit 7205dbcdec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 221 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -8,6 +8,7 @@ reports
.classpath
.project
*~
.vscode
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*

2
cmd/getConfig.go
View File

@ -86,8 +86,6 @@ func generateConfig() error {
return errors.Wrap(err, "getting step config failed")
}
//ToDo: Check for mandatory parameters
myConfigJSON, _ := config.GetJSON(stepConfig.Config)
fmt.Println(myConfigJSON)

1
go.mod
View File

@ -4,6 +4,7 @@ go 1.13
require (
github.com/ghodss/yaml v1.0.0
github.com/google/go-cmp v0.3.1
github.com/pkg/errors v0.8.1
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cobra v0.0.5

2
go.sum
View File

@ -9,6 +9,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=

19
pkg/config/config.go
View File

@ -9,6 +9,7 @@ import (
"strings"
"github.com/ghodss/yaml"
"github.com/google/go-cmp/cmp"
"github.com/pkg/errors"
)
@ -130,6 +131,22 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri
stepConfig.mixIn(flagValues, filters.Parameters)
}
// finally do the condition evaluation post processing
for _, p := range parameters {
if len(p.Conditions) > 0 {
cp := p.Conditions[0].Params[0]
dependentValue := stepConfig.Config[cp.Name]
if cmp.Equal(dependentValue, cp.Value) && stepConfig.Config[p.Name] == nil {
subMapValue := stepConfig.Config[dependentValue.(string)].(map[string]interface{})[p.Name]
if subMapValue != nil {
stepConfig.Config[p.Name] = subMapValue
} else {
stepConfig.Config[p.Name] = p.Default
}
}
}
}
return stepConfig, nil
}
@ -177,7 +194,7 @@ func (s *StepConfig) mixIn(mergeData map[string]interface{}, filter []string) {
s.Config = map[string]interface{}{}
}
s.Config = filterMap(merge(s.Config, mergeData), filter)
s.Config = merge(s.Config, filterMap(mergeData, filter))
}
func filterMap(data map[string]interface{}, filter []string) map[string]interface{} {

52
pkg/config/config_test.go
View File

@ -74,6 +74,7 @@ steps:
p4: p4_step
px4: px4_step
p5: p5_step
dependentParameter: dependentValue
stages:
stage1:
p5: p5_stage
@ -82,7 +83,7 @@ stages:
`
filters := StepFilters{
General: []string{"p0", "p1", "p2", "p3", "p4"},
Steps: []string{"p0", "p1", "p2", "p3", "p4", "p5"},
Steps: []string{"p0", "p1", "p2", "p3", "p4", "p5", "dependentParameter", "pd1", "dependentValue", "pd2"},
Stages: []string{"p0", "p1", "p2", "p3", "p4", "p5", "p6"},
Parameters: []string{"p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7"},
Env: []string{"p0", "p1", "p2", "p3", "p4", "p5"},
@ -97,6 +98,8 @@ steps:
p1: p1_step_default
px1: px1_step_default
p2: p2_step_default
dependentValue:
pd1: pd1_dependent_default
`
defaults2 := `general:
@ -112,20 +115,49 @@ steps:
defaults := []io.ReadCloser{ioutil.NopCloser(strings.NewReader(defaults1)), ioutil.NopCloser(strings.NewReader(defaults2))}
myConfig := ioutil.NopCloser(strings.NewReader(testConfig))
stepConfig, err := c.GetStepConfig(flags, paramJSON, myConfig, defaults, filters, []StepParameters{}, "stage1", "step1")
parameterMetadata := []StepParameters{
{
Name: "pd1",
Scope: []string{"STEPS"},
Conditions: []Condition{
{
Params: []Param{
{Name: "dependentParameter", Value: "dependentValue"},
},
},
},
},
{
Name: "pd2",
Default: "pd2_metadata_default",
Scope: []string{"STEPS"},
Conditions: []Condition{
{
Params: []Param{
{Name: "dependentParameter", Value: "dependentValue"},
},
},
},
},
}
stepConfig, err := c.GetStepConfig(flags, paramJSON, myConfig, defaults, filters, parameterMetadata, "stage1", "step1")
assert.Equal(t, nil, err, "error occured but none expected")
t.Run("Config", func(t *testing.T) {
expected := map[string]string{
"p0": "p0_general_default",
"p1": "p1_step_default",
"p2": "p2_general_default",
"p3": "p3_general",
"p4": "p4_step",
"p5": "p5_stage",
"p6": "p6_param",
"p7": "p7_flag",
"p0": "p0_general_default",
"p1": "p1_step_default",
"p2": "p2_general_default",
"p3": "p3_general",
"p4": "p4_step",
"p5": "p5_stage",
"p6": "p6_param",
"p7": "p7_flag",
"pd1": "pd1_dependent_default",
"pd2": "pd2_metadata_default",
}
for k, v := range expected {
t.Run(k, func(t *testing.T) {

161
pkg/config/stepmeta.go
View File

@ -48,6 +48,7 @@ type StepParameters struct {
Mandatory bool `json:"mandatory,omitempty"`
Default interface{} `json:"default,omitempty"`
Aliases []Alias `json:"aliases,omitempty"`
Conditions []Condition `json:"conditions,omitempty"`
}
// Alias defines a step input parameter alias
@ -58,9 +59,10 @@ type Alias struct {
// StepResources defines the resources to be provided by the step context, e.g. Jenkins pipeline
type StepResources struct {
Name string `json:"name"`
Description string `json:"description,omitempty"`
Type string `json:"type,omitempty"`
Name string `json:"name"`
Description string `json:"description,omitempty"`
Type string `json:"type,omitempty"`
Conditions []Condition `json:"conditions,omitempty"`
}
// StepSecrets defines the secrets to be provided by the step context, e.g. Jenkins pipeline
@ -78,14 +80,15 @@ type StepSecrets struct {
// Container defines an execution container
type Container struct {
//ToDo: check dockerOptions, dockerVolumeBind, containerPortMappings, sidecarOptions, sidecarVolumeBind
Command []string `json:"command"`
EnvVars []EnvVar `json:"env"`
Image string `json:"image"`
ImagePullPolicy string `json:"imagePullPolicy"`
Name string `json:"name"`
ReadyCommand string `json:"readyCommand"`
Shell string `json:"shell"`
WorkingDir string `json:"workingDir"`
Command []string `json:"command"`
EnvVars []EnvVar `json:"env"`
Image string `json:"image"`
ImagePullPolicy string `json:"imagePullPolicy"`
Name string `json:"name"`
ReadyCommand string `json:"readyCommand"`
Shell string `json:"shell"`
WorkingDir string `json:"workingDir"`
Conditions []Condition `json:"conditions,omitempty"`
}
// EnvVar defines an environment variable
@ -94,6 +97,18 @@ type EnvVar struct {
Value string `json:"value"`
}
// Condition defines an condition which decides when the parameter, resource or container is valid
type Condition struct {
ConditionRef string `json:"conditionRef"`
Params []Param `json:"params"`
}
// Param defines the parameters serving as inputs to the condition
type Param struct {
Name string `json:"name"`
Value string `json:"value"`
}
// StepFilters defines the filter parameters for the different sections
type StepFilters struct {
All []string
@ -123,19 +138,25 @@ func (m *StepData) ReadPipelineStepData(metadata io.ReadCloser) error {
func (m *StepData) GetParameterFilters() StepFilters {
var filters StepFilters
for _, param := range m.Spec.Inputs.Parameters {
filters.All = append(filters.All, param.Name)
parameterKeys := []string{param.Name}
for _, condition := range param.Conditions {
for _, dependentParam := range condition.Params {
parameterKeys = append(parameterKeys, dependentParam.Value)
}
}
filters.All = append(filters.All, parameterKeys...)
for _, scope := range param.Scope {
switch scope {
case "GENERAL":
filters.General = append(filters.General, param.Name)
filters.General = append(filters.General, parameterKeys...)
case "STEPS":
filters.Steps = append(filters.Steps, param.Name)
filters.Steps = append(filters.Steps, parameterKeys...)
case "STAGES":
filters.Stages = append(filters.Stages, param.Name)
filters.Stages = append(filters.Stages, parameterKeys...)
case "PARAMETERS":
filters.Parameters = append(filters.Parameters, param.Name)
filters.Parameters = append(filters.Parameters, parameterKeys...)
case "ENV":
filters.Env = append(filters.Env, param.Name)
filters.Env = append(filters.Env, parameterKeys...)
}
}
}
@ -156,7 +177,15 @@ func (m *StepData) GetContextParameterFilters() StepFilters {
containerFilters := []string{}
if len(m.Spec.Containers) > 0 {
containerFilters = append(containerFilters, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}...)
parameterKeys := []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}
for _, container := range m.Spec.Containers {
for _, condition := range container.Conditions {
for _, dependentParam := range condition.Params {
parameterKeys = append(parameterKeys, dependentParam.Value)
}
}
}
containerFilters = append(containerFilters, parameterKeys...)
}
if len(m.Spec.Sidecars) > 0 {
//ToDo: support fallback for "dockerName" configuration property -> via aliasing?
@ -175,59 +204,93 @@ func (m *StepData) GetContextParameterFilters() StepFilters {
// It only supports scenarios with one container and optionally one sidecar
func (m *StepData) GetContextDefaults(stepName string) (io.ReadCloser, error) {
p := map[string]interface{}{}
//ToDo error handling empty Containers/Sidecars
//ToDo handle empty Command
root := map[string]interface{}{}
if len(m.Spec.Containers) > 0 {
if len(m.Spec.Containers[0].Command) > 0 {
p["containerCommand"] = m.Spec.Containers[0].Command[0]
}
p["containerName"] = m.Spec.Containers[0].Name
p["containerShell"] = m.Spec.Containers[0].Shell
p["dockerEnvVars"] = envVarsAsStringSlice(m.Spec.Containers[0].EnvVars)
p["dockerImage"] = m.Spec.Containers[0].Image
p["dockerName"] = m.Spec.Containers[0].Name
p["dockerPullImage"] = m.Spec.Containers[0].ImagePullPolicy != "Never"
p["dockerWorkspace"] = m.Spec.Containers[0].WorkingDir
for _, container := range m.Spec.Containers {
key := ""
if len(container.Conditions) > 0 {
key = container.Conditions[0].Params[0].Value
}
p := map[string]interface{}{}
if key != "" {
root[key] = p
} else {
p = root
}
if len(container.Command) > 0 {
p["containerCommand"] = container.Command[0]
}
p["containerName"] = container.Name
p["containerShell"] = container.Shell
p["dockerEnvVars"] = envVarsAsStringSlice(container.EnvVars)
p["dockerImage"] = container.Image
p["dockerName"] = container.Name
p["dockerPullImage"] = container.ImagePullPolicy != "Never"
p["dockerWorkspace"] = container.WorkingDir
// Ready command not relevant for main runtime container so far
//p[] = container.ReadyCommand
}
// Ready command not relevant for main runtime container so far
//p[] = m.Spec.Containers[0].ReadyCommand
}
if len(m.Spec.Sidecars) > 0 {
if len(m.Spec.Sidecars[0].Command) > 0 {
p["sidecarCommand"] = m.Spec.Sidecars[0].Command[0]
root["sidecarCommand"] = m.Spec.Sidecars[0].Command[0]
}
p["sidecarEnvVars"] = envVarsAsStringSlice(m.Spec.Sidecars[0].EnvVars)
p["sidecarImage"] = m.Spec.Sidecars[0].Image
p["sidecarName"] = m.Spec.Sidecars[0].Name
p["sidecarPullImage"] = m.Spec.Sidecars[0].ImagePullPolicy != "Never"
p["sidecarReadyCommand"] = m.Spec.Sidecars[0].ReadyCommand
p["sidecarWorkspace"] = m.Spec.Sidecars[0].WorkingDir
root["sidecarEnvVars"] = envVarsAsStringSlice(m.Spec.Sidecars[0].EnvVars)
root["sidecarImage"] = m.Spec.Sidecars[0].Image
root["sidecarName"] = m.Spec.Sidecars[0].Name
root["sidecarPullImage"] = m.Spec.Sidecars[0].ImagePullPolicy != "Never"
root["sidecarReadyCommand"] = m.Spec.Sidecars[0].ReadyCommand
root["sidecarWorkspace"] = m.Spec.Sidecars[0].WorkingDir
}
// not filled for now since this is not relevant in Kubernetes case
//p["dockerOptions"] = m.Spec.Containers[0].
//p["dockerVolumeBind"] = m.Spec.Containers[0].
//p["containerPortMappings"] = m.Spec.Sidecars[0].
//p["sidecarOptions"] = m.Spec.Sidecars[0].
//p["sidecarVolumeBind"] = m.Spec.Sidecars[0].
//p["dockerOptions"] = container.
//p["dockerVolumeBind"] = container.
//root["containerPortMappings"] = m.Spec.Sidecars[0].
//root["sidecarOptions"] = m.Spec.Sidecars[0].
//root["sidecarVolumeBind"] = m.Spec.Sidecars[0].
if len(m.Spec.Inputs.Resources) > 0 {
var resources []string
keys := []string{}
resources := map[string][]string{}
for _, resource := range m.Spec.Inputs.Resources {
if resource.Type == "stash" {
resources = append(resources, resource.Name)
key := ""
if len(resource.Conditions) > 0 {
key = resource.Conditions[0].Params[0].Value
}
if resources[key] == nil {
keys = append(keys, key)
resources[key] = []string{}
}
resources[key] = append(resources[key], resource.Name)
}
}
for _, key := range keys {
if key == "" {
root["stashContent"] = resources[""]
} else {
if root[key] == nil {
root[key] = map[string]interface{}{
"stashContent": resources[key],
}
} else {
p := root[key].(map[string]interface{})
p["stashContent"] = resources[key]
}
}
}
p["stashContent"] = resources
}
c := Config{
Steps: map[string]map[string]interface{}{
stepName: p,
stepName: root,
},
}

59
pkg/config/stepmeta_test.go
View File

@ -68,6 +68,7 @@ func TestGetParameterFilters(t *testing.T) {
{Name: "paramFour", Scope: []string{"PARAMETERS", "ENV"}},
{Name: "paramFive", Scope: []string{"ENV"}},
{Name: "paramSix"},
{Name: "paramSeven", Scope: []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"}, Conditions: []Condition{{Params: []Param{{Name: "buildTool", Value: "mta"}}}}},
},
},
},
@ -113,17 +114,17 @@ func TestGetParameterFilters(t *testing.T) {
}{
{
Metadata: metadata1,
ExpectedGeneral: []string{"paramOne"},
ExpectedSteps: []string{"paramOne", "paramTwo"},
ExpectedStages: []string{"paramOne", "paramTwo", "paramThree"},
ExpectedParameters: []string{"paramOne", "paramTwo", "paramThree", "paramFour"},
ExpectedEnv: []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramFive"},
ExpectedAll: []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramFive", "paramSix"},
ExpectedGeneral: []string{"paramOne", "paramSeven", "mta"},
ExpectedSteps: []string{"paramOne", "paramTwo", "paramSeven", "mta"},
ExpectedStages: []string{"paramOne", "paramTwo", "paramThree", "paramSeven", "mta"},
ExpectedParameters: []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramSeven", "mta"},
ExpectedEnv: []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramFive", "paramSeven", "mta"},
ExpectedAll: []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramFive", "paramSix", "paramSeven", "mta"},
NotExpectedGeneral: []string{"paramTwo", "paramThree", "paramFour", "paramFive", "paramSix"},
NotExpectedSteps: []string{"paramThree", "paramFour", "paramFive", "paramSix"},
NotExpectedStages: []string{"paramFour", "paramFive", "paramSix"},
NotExpectedParameters: []string{"paramFive", "paramSix"},
NotExpectedEnv: []string{"paramSix"},
NotExpectedEnv: []string{"paramSix", "mta"},
NotExpectedAll: []string{},
},
{
@ -234,6 +235,11 @@ func TestGetContextParameterFilters(t *testing.T) {
Spec: StepSpec{
Containers: []Container{
{Name: "testcontainer"},
{Conditions: []Condition{
{Params: []Param{
{Name: "scanType", Value: "pip"},
}},
}},
},
},
}
@ -258,12 +264,12 @@ func TestGetContextParameterFilters(t *testing.T) {
t.Run("Containers", func(t *testing.T) {
filters := metadata2.GetContextParameterFilters()
assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.All, "incorrect filter All")
assert.NotEqual(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.General, "incorrect filter General")
assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.Steps, "incorrect filter Steps")
assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.Stages, "incorrect filter Stages")
assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.Parameters, "incorrect filter Parameters")
assert.NotEqual(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.Env, "incorrect filter Env")
assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.All, "incorrect filter All")
assert.NotEqual(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.General, "incorrect filter General")
assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.Steps, "incorrect filter Steps")
assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.Stages, "incorrect filter Stages")
assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.Parameters, "incorrect filter Parameters")
assert.NotEqual(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.Env, "incorrect filter Env")
})
t.Run("Sidecars", func(t *testing.T) {
@ -287,15 +293,38 @@ func TestGetContextDefaults(t *testing.T) {
{
Name: "buildDescriptor",
Type: "stash",
Conditions: []Condition{
{Params: []Param{
{Name: "scanType", Value: "abc"},
}},
},
},
{
Name: "source",
Type: "stash",
Conditions: []Condition{
{Params: []Param{
{Name: "scanType", Value: "abc"},
}},
},
},
{
Name: "test",
Type: "nonce",
},
{
Name: "test2",
Type: "stash",
Conditions: []Condition{
{Params: []Param{
{Name: "scanType", Value: "def"},
}},
},
},
{
Name: "test3",
Type: "stash",
},
},
},
Containers: []Container{
@ -339,7 +368,9 @@ func TestGetContextDefaults(t *testing.T) {
var d PipelineDefaults
d.ReadPipelineDefaults([]io.ReadCloser{cd})
assert.Equal(t, []interface{}{"buildDescriptor", "source"}, d.Defaults[0].Steps["testStep"]["stashContent"], "stashContent default not available")
assert.Equal(t, []interface{}{"buildDescriptor", "source"}, d.Defaults[0].Steps["testStep"]["abc"].(map[string]interface{})["stashContent"], "stashContent default not available")
assert.Equal(t, []interface{}{"test2"}, d.Defaults[0].Steps["testStep"]["def"].(map[string]interface{})["stashContent"], "stashContent default not available")
assert.Equal(t, []interface{}{"test3"}, d.Defaults[0].Steps["testStep"]["stashContent"], "stashContent default not available")
assert.Equal(t, "test/command", d.Defaults[0].Steps["testStep"]["containerCommand"], "containerCommand default not available")
assert.Equal(t, "testcontainer", d.Defaults[0].Steps["testStep"]["containerName"], "containerName default not available")
assert.Equal(t, "/bin/bash", d.Defaults[0].Steps["testStep"]["containerShell"], "containerShell default not available")