Consider custom defaults in config.yml (#995)

* Consider custom defaults in config.yml

cmd/getConfig.go

@@ -25,7 +25,7 @@ var configOptions configCommandOptions
 // ConfigCommand is the entry command for loading the configuration of a pipeline step
 func ConfigCommand() *cobra.Command {
 
-	configOptions.openFile = OpenPiperFile
+	configOptions.openFile = config.OpenPiperFile
 	var createConfigCmd = &cobra.Command{
 		Use:   "getConfig",
 		Short: "Loads the project 'Piper' configuration respecting defaults and parameters.",
@@ -69,7 +69,8 @@ func generateConfig() error {
 
 	for _, f := range GeneralConfig.DefaultConfig {
 		fc, err := configOptions.openFile(f)
-		if err != nil {
+		// only create error for non-default values
+		if err != nil && f != ".pipeline/defaults.yaml" {
 			return errors.Wrapf(err, "config: getting defaults failed: '%v'", f)
 		}
 		defaultConfig = append(defaultConfig, fc)

cmd/githubPublishRelease_generated.go

@@ -48,7 +48,7 @@ The result looks like
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			log.SetStepName("githubPublishRelease")
 			log.SetVerbose(GeneralConfig.Verbose)
-			return PrepareConfig(cmd, &metadata, "githubPublishRelease", &myGithubPublishReleaseOptions, OpenPiperFile)
+			return PrepareConfig(cmd, &metadata, "githubPublishRelease", &myGithubPublishReleaseOptions, config.OpenPiperFile)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return githubPublishRelease(myGithubPublishReleaseOptions)

cmd/karmaExecuteTests_generated.go

@@ -35,7 +35,7 @@ In the Docker network, the containers can be referenced by the values provided i
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			log.SetStepName("karmaExecuteTests")
 			log.SetVerbose(GeneralConfig.Verbose)
-			return PrepareConfig(cmd, &metadata, "karmaExecuteTests", &myKarmaExecuteTestsOptions, OpenPiperFile)
+			return PrepareConfig(cmd, &metadata, "karmaExecuteTests", &myKarmaExecuteTestsOptions, config.OpenPiperFile)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return karmaExecuteTests(myKarmaExecuteTestsOptions)

cmd/piper.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"strings"
 
 	"github.com/SAP/jenkins-library/pkg/config"
 	"github.com/SAP/jenkins-library/pkg/log"
@@ -57,7 +56,7 @@ func Execute() {
 func addRootFlags(rootCmd *cobra.Command) {
 
 	rootCmd.PersistentFlags().StringVar(&GeneralConfig.CustomConfig, "customConfig", ".pipeline/config.yml", "Path to the pipeline configuration file")
-	rootCmd.PersistentFlags().StringSliceVar(&GeneralConfig.DefaultConfig, "defaultConfig", nil, "Default configurations, passed as path to yaml file")
+	rootCmd.PersistentFlags().StringSliceVar(&GeneralConfig.DefaultConfig, "defaultConfig", []string{".pipeline/defaults.yaml"}, "Default configurations, passed as path to yaml file")
 	rootCmd.PersistentFlags().StringVar(&GeneralConfig.ParametersJSON, "parametersJSON", os.Getenv("PIPER_parametersJSON"), "Parameters to be considered in JSON format")
 	rootCmd.PersistentFlags().StringVar(&GeneralConfig.StageName, "stageName", os.Getenv("STAGE_NAME"), "Name of the stage for which configuration should be included")
 	rootCmd.PersistentFlags().StringVar(&GeneralConfig.StepConfigJSON, "stepConfigJSON", os.Getenv("PIPER_stepConfigJSON"), "Step configuration in JSON format")
@@ -112,12 +111,3 @@ func PrepareConfig(cmd *cobra.Command, metadata *config.StepData, stepName strin
 
 	return nil
 }
-
-// OpenPiperFile provides functionality to retrieve configuration via file or http
-func OpenPiperFile(name string) (io.ReadCloser, error) {
-	//ToDo: support also https as source
-	if !strings.HasPrefix(name, "http") {
-		return os.Open(name)
-	}
-	return nil, fmt.Errorf("file location not yet supported for '%v'", name)
-}

cmd/piper_test.go

@@ -13,10 +13,10 @@ import (
 )
 
 type execMockRunner struct {
-	dir   []string
+	dir            []string
-	calls []execCall
+	calls          []execCall
-	stdout io.Writer
+	stdout         io.Writer
-	stderr io.Writer
+	stderr         io.Writer
 	shouldFailWith error
 }
 
@@ -26,10 +26,10 @@ type execCall struct {
 }
 
 type shellMockRunner struct {
-	dir   string
+	dir            string
-	calls []string
+	calls          []string
-	stdout io.Writer
+	stdout         io.Writer
-	stderr io.Writer
+	stderr         io.Writer
 	shouldFailWith error
 }
 
@@ -46,16 +46,14 @@ func (m *execMockRunner) RunExecutable(e string, p ...string) error {
 	return nil
 }
 
-func (m * execMockRunner) Stdout(out io.Writer) {
+func (m *execMockRunner) Stdout(out io.Writer) {
 	m.stdout = out
 }
 
-
+func (m *execMockRunner) Stderr(err io.Writer) {
-func (m * execMockRunner) Stderr(err io.Writer) {
 	m.stderr = err
 }
 
-
 func (m *shellMockRunner) Dir(d string) {
 	m.dir = d
 }
@@ -70,12 +68,11 @@ func (m *shellMockRunner) RunShell(s string, c string) error {
 	return nil
 }
 
-func (m * shellMockRunner) Stdout(out io.Writer) {
+func (m *shellMockRunner) Stdout(out io.Writer) {
 	m.stdout = out
 }
 
-
+func (m *shellMockRunner) Stderr(err io.Writer) {
-func (m * shellMockRunner) Stderr(err io.Writer) {
 	m.stderr = err
 }
 

cmd/version_generated.go

@@ -22,7 +22,7 @@ func VersionCommand() *cobra.Command {
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			log.SetStepName("version")
 			log.SetVerbose(GeneralConfig.Verbose)
-			return PrepareConfig(cmd, &metadata, "version", &myVersionOptions, OpenPiperFile)
+			return PrepareConfig(cmd, &metadata, "version", &myVersionOptions, config.OpenPiperFile)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return version(myVersionOptions)

pkg/config/config.go

@@ -15,9 +15,11 @@ import (
 
 // Config defines the structure of the config files
 type Config struct {
-	General map[string]interface{}            `json:"general"`
+	CustomDefaults []string                          `json:"customDefaults,omitempty"`
-	Stages  map[string]map[string]interface{} `json:"stages"`
+	General        map[string]interface{}            `json:"general"`
-	Steps   map[string]map[string]interface{} `json:"steps"`
+	Stages         map[string]map[string]interface{} `json:"stages"`
+	Steps          map[string]map[string]interface{} `json:"steps"`
+	openFile       func(s string) (io.ReadCloser, error)
 }
 
 // StepConfig defines the structure for merged step configuration
@@ -55,7 +57,7 @@ func (c *Config) ApplyAliasConfig(parameters []StepParameters, filters StepFilte
 }
 
 func setParamValueFromAlias(configMap map[string]interface{}, filter []string, p StepParameters) map[string]interface{} {
-	if configMap[p.Name] == nil && sliceContains(filter, p.Name) {
+	if configMap != nil && configMap[p.Name] == nil && sliceContains(filter, p.Name) {
 		for _, a := range p.Aliases {
 			configMap[p.Name] = getDeepAliasValue(configMap, a.Name)
 			if configMap[p.Name] != nil {
@@ -89,6 +91,20 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri
 	}
 	c.ApplyAliasConfig(parameters, filters, stageName, stepName)
 
+	// consider custom defaults defined in config.yml
+	if c.CustomDefaults != nil && len(c.CustomDefaults) > 0 {
+		if c.openFile == nil {
+			c.openFile = OpenPiperFile
+		}
+		for _, f := range c.CustomDefaults {
+			fc, err := c.openFile(f)
+			if err != nil {
+				return StepConfig{}, errors.Wrapf(err, "getting default '%v' failed", f)
+			}
+			defaults = append(defaults, fc)
+		}
+	}
+
 	if err := d.ReadPipelineDefaults(defaults); err != nil {
 		switch err.(type) {
 		case *ParseError:
@@ -178,6 +194,15 @@ func GetJSON(data interface{}) (string, error) {
 	return string(result), nil
 }
 
+// OpenPiperFile provides functionality to retrieve configuration via file or http
+func OpenPiperFile(name string) (io.ReadCloser, error) {
+	//ToDo: support also https as source
+	if !strings.HasPrefix(name, "http") {
+		return os.Open(name)
+	}
+	return nil, fmt.Errorf("file location not yet supported for '%v'", name)
+}
+
 func envValues(filter []string) map[string]interface{} {
 	vals := map[string]interface{}{}
 	for _, param := range filter {

pkg/config/config_test.go

@@ -21,6 +21,10 @@ func (errReadCloser) Close() error {
 	return nil
 }
 
+func customDefaultsOpenFileMock(name string) (io.ReadCloser, error) {
+	return ioutil.NopCloser(strings.NewReader("general:\n  p0: p0_custom_default")), nil
+}
+
 func TestReadConfig(t *testing.T) {
 
 	var c Config
@@ -107,6 +111,7 @@ steps:
   px2: px2_general_default
   p3: p3_general_default 
 `
+
 		paramJSON := `{"p6":"p6_param","p7":"p7_param"}`
 
 		flags := map[string]interface{}{"p7": "p7_flag"}
@@ -180,6 +185,19 @@ steps:
 		})
 	})
 
+	t.Run("Consider custom defaults from config", func(t *testing.T) {
+		var c Config
+		testConfDefaults := "customDefaults:\n- testDefaults.yaml"
+
+		c.openFile = customDefaultsOpenFileMock
+
+		stepConfig, err := c.GetStepConfig(nil, "", ioutil.NopCloser(strings.NewReader(testConfDefaults)), nil, StepFilters{General: []string{"p0"}}, nil, "stage1", "step1")
+
+		assert.NoError(t, err, "Error occured but no error expected")
+		assert.Equal(t, "p0_custom_default", stepConfig.Config["p0"])
+
+	})
+
 	t.Run("Failure case config", func(t *testing.T) {
 		var c Config
 		myConfig := ioutil.NopCloser(strings.NewReader("invalid config"))

pkg/generator/helper/helper.go

@@ -56,7 +56,7 @@ func {{.CobraCmdFuncName}}() *cobra.Command {
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			log.SetStepName("{{ .StepName }}")
 			log.SetVerbose({{if .ExportPrefix}}{{ .ExportPrefix }}.{{end}}GeneralConfig.Verbose)
-			return {{if .ExportPrefix}}{{ .ExportPrefix }}.{{end}}PrepareConfig(cmd, &metadata, "{{ .StepName }}", &my{{ .StepName | title}}Options, {{if .ExportPrefix}}{{ .ExportPrefix }}.{{end}}OpenPiperFile)
+			return {{if .ExportPrefix}}{{ .ExportPrefix }}.{{end}}PrepareConfig(cmd, &metadata, "{{ .StepName }}", &my{{ .StepName | title}}Options, config.OpenPiperFile)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return {{.StepName}}(my{{ .StepName | title }}Options)

pkg/generator/helper/testdata/TestProcessMetaFiles/step_code_generated.golden

@@ -28,7 +28,7 @@ func TestStepCommand() *cobra.Command {
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			log.SetStepName("testStep")
 			log.SetVerbose(GeneralConfig.Verbose)
-			return PrepareConfig(cmd, &metadata, "testStep", &myTestStepOptions, OpenPiperFile)
+			return PrepareConfig(cmd, &metadata, "testStep", &myTestStepOptions, config.OpenPiperFile)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return testStep(myTestStepOptions)