2019-10-22 15:41:27 +02:00
package config
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
2019-10-29 11:58:24 +02:00
"strings"
2020-03-19 18:24:35 +02:00
2020-03-31 08:47:09 +02:00
"github.com/SAP/jenkins-library/pkg/http"
2020-03-19 18:24:35 +02:00
"github.com/SAP/jenkins-library/pkg/log"
"github.com/ghodss/yaml"
"github.com/google/go-cmp/cmp"
"github.com/pkg/errors"
2019-10-22 15:41:27 +02:00
)
// Config defines the structure of the config files
type Config struct {
2019-11-21 17:09:57 +02:00
CustomDefaults [ ] string ` json:"customDefaults,omitempty" `
General map [ string ] interface { } ` json:"general" `
Stages map [ string ] map [ string ] interface { } ` json:"stages" `
Steps map [ string ] map [ string ] interface { } ` json:"steps" `
2020-05-05 08:36:24 +02:00
Hooks map [ string ] * json . RawMessage ` json:"hooks,omitempty" `
2019-11-21 17:09:57 +02:00
openFile func ( s string ) ( io . ReadCloser , error )
2019-10-22 15:41:27 +02:00
}
// StepConfig defines the structure for merged step configuration
type StepConfig struct {
2020-05-05 08:36:24 +02:00
Config map [ string ] interface { }
HookConfig map [ string ] * json . RawMessage
2019-10-22 15:41:27 +02:00
}
// ReadConfig loads config and returns its content
func ( c * Config ) ReadConfig ( configuration io . ReadCloser ) error {
defer configuration . Close ( )
content , err := ioutil . ReadAll ( configuration )
if err != nil {
return errors . Wrapf ( err , "error reading %v" , configuration )
}
err = yaml . Unmarshal ( content , & c )
if err != nil {
2020-04-09 13:40:04 +02:00
return NewParseError ( fmt . Sprintf ( "format of configuration is invalid %q: %v" , content , err ) )
2019-10-22 15:41:27 +02:00
}
return nil
}
2019-10-29 11:58:24 +02:00
// ApplyAliasConfig adds configuration values available on aliases to primary configuration parameters
2020-04-01 20:46:33 +02:00
func ( c * Config ) ApplyAliasConfig ( parameters [ ] StepParameters , secrets [ ] StepSecrets , filters StepFilters , stageName , stepName string , stepAliases [ ] Alias ) {
2020-03-19 18:24:35 +02:00
// copy configuration from step alias to correct step
if len ( stepAliases ) > 0 {
c . copyStepAliasConfig ( stepName , stepAliases )
}
2019-10-29 11:58:24 +02:00
for _ , p := range parameters {
2020-04-01 20:46:33 +02:00
c . General = setParamValueFromAlias ( c . General , filters . General , p . Name , p . Aliases )
2019-10-29 11:58:24 +02:00
if c . Stages [ stageName ] != nil {
2020-04-01 20:46:33 +02:00
c . Stages [ stageName ] = setParamValueFromAlias ( c . Stages [ stageName ] , filters . Stages , p . Name , p . Aliases )
2019-10-29 11:58:24 +02:00
}
if c . Steps [ stepName ] != nil {
2020-04-01 20:46:33 +02:00
c . Steps [ stepName ] = setParamValueFromAlias ( c . Steps [ stepName ] , filters . Steps , p . Name , p . Aliases )
}
}
for _ , s := range secrets {
c . General = setParamValueFromAlias ( c . General , filters . General , s . Name , s . Aliases )
if c . Stages [ stageName ] != nil {
c . Stages [ stageName ] = setParamValueFromAlias ( c . Stages [ stageName ] , filters . Stages , s . Name , s . Aliases )
}
if c . Steps [ stepName ] != nil {
c . Steps [ stepName ] = setParamValueFromAlias ( c . Steps [ stepName ] , filters . Steps , s . Name , s . Aliases )
2019-10-29 11:58:24 +02:00
}
}
}
2020-04-01 20:46:33 +02:00
func setParamValueFromAlias ( configMap map [ string ] interface { } , filter [ ] string , name string , aliases [ ] Alias ) map [ string ] interface { } {
if configMap != nil && configMap [ name ] == nil && sliceContains ( filter , name ) {
for _ , a := range aliases {
2019-11-22 11:30:44 +02:00
aliasVal := getDeepAliasValue ( configMap , a . Name )
if aliasVal != nil {
2020-04-01 20:46:33 +02:00
configMap [ name ] = aliasVal
if a . Deprecated {
log . Entry ( ) . WithField ( "package" , "SAP/jenkins-library/pkg/config" ) . Warningf ( "DEPRECATION NOTICE: old step config key '%v' used. Please switch to '%v'!" , a . Name , name )
}
2019-11-22 11:30:44 +02:00
}
2020-04-01 20:46:33 +02:00
if configMap [ name ] != nil {
2019-10-29 11:58:24 +02:00
return configMap
}
}
}
return configMap
}
func getDeepAliasValue ( configMap map [ string ] interface { } , key string ) interface { } {
parts := strings . Split ( key , "/" )
if len ( parts ) > 1 {
if configMap [ parts [ 0 ] ] == nil {
return nil
}
return getDeepAliasValue ( configMap [ parts [ 0 ] ] . ( map [ string ] interface { } ) , strings . Join ( parts [ 1 : ] , "/" ) )
}
return configMap [ key ]
}
2020-03-19 18:24:35 +02:00
func ( c * Config ) copyStepAliasConfig ( stepName string , stepAliases [ ] Alias ) {
for _ , stepAlias := range stepAliases {
if c . Steps [ stepAlias . Name ] != nil {
if stepAlias . Deprecated {
log . Entry ( ) . WithField ( "package" , "SAP/jenkins-library/pkg/config" ) . Warningf ( "DEPRECATION NOTICE: old step configuration used for step '%v'. Please switch to '%v'!" , stepAlias . Name , stepName )
}
for paramName , paramValue := range c . Steps [ stepAlias . Name ] {
if c . Steps [ stepName ] == nil {
c . Steps [ stepName ] = map [ string ] interface { } { }
}
if c . Steps [ stepName ] [ paramName ] == nil {
c . Steps [ stepName ] [ paramName ] = paramValue
}
}
}
}
}
2019-10-22 15:41:27 +02:00
// GetStepConfig provides merged step configuration using defaults, config, if available
2020-05-14 10:50:58 +02:00
func ( c * Config ) GetStepConfig ( flagValues map [ string ] interface { } , paramJSON string , configuration io . ReadCloser , defaults [ ] io . ReadCloser , ignoreCustomDefaults bool , filters StepFilters , parameters [ ] StepParameters , secrets [ ] StepSecrets , envParameters map [ string ] interface { } , stageName , stepName string , stepAliases [ ] Alias ) ( StepConfig , error ) {
2019-10-22 15:41:27 +02:00
var stepConfig StepConfig
var d PipelineDefaults
2019-11-06 11:28:15 +02:00
if configuration != nil {
if err := c . ReadConfig ( configuration ) ; err != nil {
2019-10-22 15:41:27 +02:00
return StepConfig { } , errors . Wrap ( err , "failed to parse custom pipeline configuration" )
}
}
2020-03-19 18:24:35 +02:00
2020-04-01 20:46:33 +02:00
c . ApplyAliasConfig ( parameters , secrets , filters , stageName , stepName , stepAliases )
2019-10-22 15:41:27 +02:00
2020-05-14 10:50:58 +02:00
// consider custom defaults defined in config.yml unless told otherwise
if ignoreCustomDefaults {
log . Entry ( ) . Info ( "Ignoring custom defaults from pipeline config" )
} else if c . CustomDefaults != nil && len ( c . CustomDefaults ) > 0 {
2019-11-21 17:09:57 +02:00
if c . openFile == nil {
c . openFile = OpenPiperFile
}
for _ , f := range c . CustomDefaults {
fc , err := c . openFile ( f )
if err != nil {
return StepConfig { } , errors . Wrapf ( err , "getting default '%v' failed" , f )
}
defaults = append ( defaults , fc )
}
}
2019-10-22 15:41:27 +02:00
if err := d . ReadPipelineDefaults ( defaults ) ; err != nil {
2020-03-31 08:47:09 +02:00
return StepConfig { } , errors . Wrap ( err , "failed to read default configuration" )
2019-10-22 15:41:27 +02:00
}
2019-11-22 11:30:44 +02:00
// initialize with defaults from step.yaml
stepConfig . mixInStepDefaults ( parameters )
// read defaults & merge general -> steps (-> general -> steps ...)
2019-10-22 15:41:27 +02:00
for _ , def := range d . Defaults {
2020-04-01 20:46:33 +02:00
def . ApplyAliasConfig ( parameters , secrets , filters , stageName , stepName , stepAliases )
2019-10-22 15:41:27 +02:00
stepConfig . mixIn ( def . General , filters . General )
stepConfig . mixIn ( def . Steps [ stepName ] , filters . Steps )
2020-05-05 08:36:24 +02:00
// process hook configuration - this is only supported via defaults
if stepConfig . HookConfig == nil {
stepConfig . HookConfig = def . Hooks
}
2019-10-22 15:41:27 +02:00
}
2020-01-15 13:16:25 +02:00
// merge parameters provided by Piper environment
stepConfig . mixIn ( envParameters , filters . All )
2019-11-22 11:30:44 +02:00
// read config & merge - general -> steps -> stages
2019-10-22 15:41:27 +02:00
stepConfig . mixIn ( c . General , filters . General )
stepConfig . mixIn ( c . Steps [ stepName ] , filters . Steps )
stepConfig . mixIn ( c . Stages [ stageName ] , filters . Stages )
2019-11-22 11:30:44 +02:00
// merge parameters provided via env vars
2019-10-22 15:41:27 +02:00
stepConfig . mixIn ( envValues ( filters . All ) , filters . All )
2019-11-22 11:30:44 +02:00
// if parameters are provided in JSON format merge them
2019-10-22 15:41:27 +02:00
if len ( paramJSON ) != 0 {
var params map [ string ] interface { }
2020-04-01 20:46:33 +02:00
err := json . Unmarshal ( [ ] byte ( paramJSON ) , & params )
if err != nil {
log . Entry ( ) . Warnf ( "failed to parse parameters from environment: %v" , err )
} else {
//apply aliases
for _ , p := range parameters {
params = setParamValueFromAlias ( params , filters . Parameters , p . Name , p . Aliases )
}
2019-10-29 11:58:24 +02:00
2020-04-01 20:46:33 +02:00
stepConfig . mixIn ( params , filters . Parameters )
2019-10-29 11:58:24 +02:00
}
2019-10-22 15:41:27 +02:00
}
2019-11-22 11:30:44 +02:00
// merge command line flags
2019-10-22 15:41:27 +02:00
if flagValues != nil {
stepConfig . mixIn ( flagValues , filters . Parameters )
}
2019-11-05 17:30:41 +02:00
// finally do the condition evaluation post processing
for _ , p := range parameters {
if len ( p . Conditions ) > 0 {
cp := p . Conditions [ 0 ] . Params [ 0 ]
dependentValue := stepConfig . Config [ cp . Name ]
if cmp . Equal ( dependentValue , cp . Value ) && stepConfig . Config [ p . Name ] == nil {
subMapValue := stepConfig . Config [ dependentValue . ( string ) ] . ( map [ string ] interface { } ) [ p . Name ]
if subMapValue != nil {
stepConfig . Config [ p . Name ] = subMapValue
} else {
stepConfig . Config [ p . Name ] = p . Default
}
}
}
}
2019-10-22 15:41:27 +02:00
return stepConfig , nil
}
// GetStepConfigWithJSON provides merged step configuration using a provided stepConfigJSON with additional flags provided
func GetStepConfigWithJSON ( flagValues map [ string ] interface { } , stepConfigJSON string , filters StepFilters ) StepConfig {
var stepConfig StepConfig
stepConfigMap := map [ string ] interface { } { }
2020-04-01 20:46:33 +02:00
err := json . Unmarshal ( [ ] byte ( stepConfigJSON ) , & stepConfigMap )
if err != nil {
log . Entry ( ) . Warnf ( "invalid stepConfig JSON: %v" , err )
}
2019-10-22 15:41:27 +02:00
stepConfig . mixIn ( stepConfigMap , filters . All )
// ToDo: mix in parametersJSON
if flagValues != nil {
stepConfig . mixIn ( flagValues , filters . Parameters )
}
return stepConfig
}
// GetJSON returns JSON representation of an object
func GetJSON ( data interface { } ) ( string , error ) {
result , err := json . Marshal ( data )
if err != nil {
return "" , errors . Wrapf ( err , "error marshalling json: %v" , err )
}
return string ( result ) , nil
}
2019-11-21 17:09:57 +02:00
// OpenPiperFile provides functionality to retrieve configuration via file or http
func OpenPiperFile ( name string ) ( io . ReadCloser , error ) {
2020-03-31 08:47:09 +02:00
if ! strings . HasPrefix ( name , "http://" ) && ! strings . HasPrefix ( name , "https://" ) {
2019-11-21 17:09:57 +02:00
return os . Open ( name )
}
2020-03-31 08:47:09 +02:00
// support http(s) urls next to file path - url cannot be protected
client := http . Client { }
response , err := client . SendRequest ( "GET" , name , nil , nil , nil )
2020-04-01 20:46:33 +02:00
if err != nil {
return nil , err
}
return response . Body , nil
2019-11-21 17:09:57 +02:00
}
2019-10-22 15:41:27 +02:00
func envValues ( filter [ ] string ) map [ string ] interface { } {
vals := map [ string ] interface { } { }
for _ , param := range filter {
if envVal := os . Getenv ( "PIPER_" + param ) ; len ( envVal ) != 0 {
vals [ param ] = os . Getenv ( "PIPER_" + param )
}
}
return vals
}
func ( s * StepConfig ) mixIn ( mergeData map [ string ] interface { } , filter [ ] string ) {
if s . Config == nil {
s . Config = map [ string ] interface { } { }
}
2019-11-05 17:30:41 +02:00
s . Config = merge ( s . Config , filterMap ( mergeData , filter ) )
2019-10-22 15:41:27 +02:00
}
2019-11-22 11:30:44 +02:00
func ( s * StepConfig ) mixInStepDefaults ( stepParams [ ] StepParameters ) {
if s . Config == nil {
s . Config = map [ string ] interface { } { }
}
for _ , p := range stepParams {
if p . Default != nil {
s . Config [ p . Name ] = p . Default
}
}
}
2019-10-22 15:41:27 +02:00
func filterMap ( data map [ string ] interface { } , filter [ ] string ) map [ string ] interface { } {
result := map [ string ] interface { } { }
if data == nil {
data = map [ string ] interface { } { }
}
for key , value := range data {
if len ( filter ) == 0 || sliceContains ( filter , key ) {
result [ key ] = value
}
}
return result
}
func merge ( base , overlay map [ string ] interface { } ) map [ string ] interface { } {
result := map [ string ] interface { } { }
if base == nil {
base = map [ string ] interface { } { }
}
for key , value := range base {
result [ key ] = value
}
for key , value := range overlay {
if val , ok := value . ( map [ string ] interface { } ) ; ok {
if valBaseKey , ok := base [ key ] . ( map [ string ] interface { } ) ; ! ok {
result [ key ] = merge ( map [ string ] interface { } { } , val )
} else {
result [ key ] = merge ( valBaseKey , val )
}
} else {
result [ key ] = value
}
}
return result
}
func sliceContains ( slice [ ] string , find string ) bool {
for _ , elem := range slice {
if elem == find {
return true
}
}
return false
}