2019-10-22 15:41:27 +02:00
package config
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
2020-07-09 14:13:06 +02:00
"reflect"
2019-10-29 11:58:24 +02:00
"strings"
2020-03-19 18:24:35 +02:00
2020-03-31 08:47:09 +02:00
"github.com/SAP/jenkins-library/pkg/http"
2020-03-19 18:24:35 +02:00
"github.com/SAP/jenkins-library/pkg/log"
"github.com/ghodss/yaml"
"github.com/google/go-cmp/cmp"
"github.com/pkg/errors"
2019-10-22 15:41:27 +02:00
)
// Config defines the structure of the config files
type Config struct {
2020-09-16 14:50:09 +02:00
CustomDefaults [ ] string ` json:"customDefaults,omitempty" `
General map [ string ] interface { } ` json:"general" `
Stages map [ string ] map [ string ] interface { } ` json:"stages" `
Steps map [ string ] map [ string ] interface { } ` json:"steps" `
Hooks * json . RawMessage ` json:"hooks,omitempty" `
defaults PipelineDefaults
initialized bool
openFile func ( s string ) ( io . ReadCloser , error )
vaultCredentials VaultCredentials
2019-10-22 15:41:27 +02:00
}
// StepConfig defines the structure for merged step configuration
type StepConfig struct {
2020-05-05 08:36:24 +02:00
Config map [ string ] interface { }
2020-07-28 17:19:33 +02:00
HookConfig * json . RawMessage
2019-10-22 15:41:27 +02:00
}
// ReadConfig loads config and returns its content
func ( c * Config ) ReadConfig ( configuration io . ReadCloser ) error {
defer configuration . Close ( )
content , err := ioutil . ReadAll ( configuration )
if err != nil {
return errors . Wrapf ( err , "error reading %v" , configuration )
}
err = yaml . Unmarshal ( content , & c )
if err != nil {
2020-04-09 13:40:04 +02:00
return NewParseError ( fmt . Sprintf ( "format of configuration is invalid %q: %v" , content , err ) )
2019-10-22 15:41:27 +02:00
}
return nil
}
2019-10-29 11:58:24 +02:00
// ApplyAliasConfig adds configuration values available on aliases to primary configuration parameters
2020-04-01 20:46:33 +02:00
func ( c * Config ) ApplyAliasConfig ( parameters [ ] StepParameters , secrets [ ] StepSecrets , filters StepFilters , stageName , stepName string , stepAliases [ ] Alias ) {
2020-03-19 18:24:35 +02:00
// copy configuration from step alias to correct step
if len ( stepAliases ) > 0 {
c . copyStepAliasConfig ( stepName , stepAliases )
}
2019-10-29 11:58:24 +02:00
for _ , p := range parameters {
2020-04-01 20:46:33 +02:00
c . General = setParamValueFromAlias ( c . General , filters . General , p . Name , p . Aliases )
2019-10-29 11:58:24 +02:00
if c . Stages [ stageName ] != nil {
2020-04-01 20:46:33 +02:00
c . Stages [ stageName ] = setParamValueFromAlias ( c . Stages [ stageName ] , filters . Stages , p . Name , p . Aliases )
2019-10-29 11:58:24 +02:00
}
if c . Steps [ stepName ] != nil {
2020-04-01 20:46:33 +02:00
c . Steps [ stepName ] = setParamValueFromAlias ( c . Steps [ stepName ] , filters . Steps , p . Name , p . Aliases )
}
}
for _ , s := range secrets {
c . General = setParamValueFromAlias ( c . General , filters . General , s . Name , s . Aliases )
if c . Stages [ stageName ] != nil {
c . Stages [ stageName ] = setParamValueFromAlias ( c . Stages [ stageName ] , filters . Stages , s . Name , s . Aliases )
}
if c . Steps [ stepName ] != nil {
c . Steps [ stepName ] = setParamValueFromAlias ( c . Steps [ stepName ] , filters . Steps , s . Name , s . Aliases )
2019-10-29 11:58:24 +02:00
}
}
}
2020-04-01 20:46:33 +02:00
func setParamValueFromAlias ( configMap map [ string ] interface { } , filter [ ] string , name string , aliases [ ] Alias ) map [ string ] interface { } {
if configMap != nil && configMap [ name ] == nil && sliceContains ( filter , name ) {
for _ , a := range aliases {
2019-11-22 11:30:44 +02:00
aliasVal := getDeepAliasValue ( configMap , a . Name )
if aliasVal != nil {
2020-04-01 20:46:33 +02:00
configMap [ name ] = aliasVal
if a . Deprecated {
log . Entry ( ) . WithField ( "package" , "SAP/jenkins-library/pkg/config" ) . Warningf ( "DEPRECATION NOTICE: old step config key '%v' used. Please switch to '%v'!" , a . Name , name )
}
2019-11-22 11:30:44 +02:00
}
2020-04-01 20:46:33 +02:00
if configMap [ name ] != nil {
2019-10-29 11:58:24 +02:00
return configMap
}
}
}
return configMap
}
func getDeepAliasValue ( configMap map [ string ] interface { } , key string ) interface { } {
parts := strings . Split ( key , "/" )
if len ( parts ) > 1 {
if configMap [ parts [ 0 ] ] == nil {
return nil
}
2020-07-09 14:13:06 +02:00
paramValueType := reflect . ValueOf ( configMap [ parts [ 0 ] ] )
if paramValueType . Kind ( ) != reflect . Map {
log . Entry ( ) . Debugf ( "Ignoring alias '%v' as '%v' is not pointing to a map." , key , parts [ 0 ] )
return nil
}
2019-10-29 11:58:24 +02:00
return getDeepAliasValue ( configMap [ parts [ 0 ] ] . ( map [ string ] interface { } ) , strings . Join ( parts [ 1 : ] , "/" ) )
}
return configMap [ key ]
}
2020-03-19 18:24:35 +02:00
func ( c * Config ) copyStepAliasConfig ( stepName string , stepAliases [ ] Alias ) {
for _ , stepAlias := range stepAliases {
if c . Steps [ stepAlias . Name ] != nil {
if stepAlias . Deprecated {
2020-06-12 15:19:13 +02:00
log . Entry ( ) . WithField ( "package" , "SAP/jenkins-library/pkg/config" ) . Warningf ( "DEPRECATION NOTICE: step configuration available for deprecated step '%v'. Please remove or move configuration to step '%v'!" , stepAlias . Name , stepName )
2020-03-19 18:24:35 +02:00
}
for paramName , paramValue := range c . Steps [ stepAlias . Name ] {
if c . Steps [ stepName ] == nil {
c . Steps [ stepName ] = map [ string ] interface { } { }
}
if c . Steps [ stepName ] [ paramName ] == nil {
c . Steps [ stepName ] [ paramName ] = paramValue
}
}
}
}
}
2020-06-15 14:17:59 +02:00
// InitializeConfig prepares the config object, i.e. loading content, etc.
func ( c * Config ) InitializeConfig ( configuration io . ReadCloser , defaults [ ] io . ReadCloser , ignoreCustomDefaults bool ) error {
2019-11-06 11:28:15 +02:00
if configuration != nil {
if err := c . ReadConfig ( configuration ) ; err != nil {
2020-06-15 14:17:59 +02:00
return errors . Wrap ( err , "failed to parse custom pipeline configuration" )
2019-10-22 15:41:27 +02:00
}
}
2020-03-19 18:24:35 +02:00
2020-05-14 10:50:58 +02:00
// consider custom defaults defined in config.yml unless told otherwise
if ignoreCustomDefaults {
log . Entry ( ) . Info ( "Ignoring custom defaults from pipeline config" )
} else if c . CustomDefaults != nil && len ( c . CustomDefaults ) > 0 {
2019-11-21 17:09:57 +02:00
if c . openFile == nil {
c . openFile = OpenPiperFile
}
for _ , f := range c . CustomDefaults {
fc , err := c . openFile ( f )
if err != nil {
2020-06-15 14:17:59 +02:00
return errors . Wrapf ( err , "getting default '%v' failed" , f )
2019-11-21 17:09:57 +02:00
}
defaults = append ( defaults , fc )
}
}
2020-06-15 14:17:59 +02:00
if err := c . defaults . ReadPipelineDefaults ( defaults ) ; err != nil {
return errors . Wrap ( err , "failed to read default configuration" )
2019-10-22 15:41:27 +02:00
}
2020-06-15 14:17:59 +02:00
c . initialized = true
return nil
}
// GetStepConfig provides merged step configuration using defaults, config, if available
func ( c * Config ) GetStepConfig ( flagValues map [ string ] interface { } , paramJSON string , configuration io . ReadCloser , defaults [ ] io . ReadCloser , ignoreCustomDefaults bool , filters StepFilters , parameters [ ] StepParameters , secrets [ ] StepSecrets , envParameters map [ string ] interface { } , stageName , stepName string , stepAliases [ ] Alias ) ( StepConfig , error ) {
var stepConfig StepConfig
var err error
if ! c . initialized {
err = c . InitializeConfig ( configuration , defaults , ignoreCustomDefaults )
if err != nil {
return StepConfig { } , err
}
}
c . ApplyAliasConfig ( parameters , secrets , filters , stageName , stepName , stepAliases )
2019-10-22 15:41:27 +02:00
2019-11-22 11:30:44 +02:00
// initialize with defaults from step.yaml
stepConfig . mixInStepDefaults ( parameters )
// read defaults & merge general -> steps (-> general -> steps ...)
2020-06-15 14:17:59 +02:00
for _ , def := range c . defaults . Defaults {
2020-04-01 20:46:33 +02:00
def . ApplyAliasConfig ( parameters , secrets , filters , stageName , stepName , stepAliases )
2019-10-22 15:41:27 +02:00
stepConfig . mixIn ( def . General , filters . General )
stepConfig . mixIn ( def . Steps [ stepName ] , filters . Steps )
2020-06-16 09:06:37 +02:00
stepConfig . mixIn ( def . Stages [ stageName ] , filters . Steps )
2020-05-05 08:36:24 +02:00
// process hook configuration - this is only supported via defaults
if stepConfig . HookConfig == nil {
stepConfig . HookConfig = def . Hooks
}
2019-10-22 15:41:27 +02:00
}
2020-01-15 13:16:25 +02:00
// merge parameters provided by Piper environment
stepConfig . mixIn ( envParameters , filters . All )
2019-11-22 11:30:44 +02:00
// read config & merge - general -> steps -> stages
2019-10-22 15:41:27 +02:00
stepConfig . mixIn ( c . General , filters . General )
stepConfig . mixIn ( c . Steps [ stepName ] , filters . Steps )
stepConfig . mixIn ( c . Stages [ stageName ] , filters . Stages )
2019-11-22 11:30:44 +02:00
// merge parameters provided via env vars
2019-10-22 15:41:27 +02:00
stepConfig . mixIn ( envValues ( filters . All ) , filters . All )
2019-11-22 11:30:44 +02:00
// if parameters are provided in JSON format merge them
2019-10-22 15:41:27 +02:00
if len ( paramJSON ) != 0 {
var params map [ string ] interface { }
2020-04-01 20:46:33 +02:00
err := json . Unmarshal ( [ ] byte ( paramJSON ) , & params )
if err != nil {
log . Entry ( ) . Warnf ( "failed to parse parameters from environment: %v" , err )
} else {
//apply aliases
for _ , p := range parameters {
params = setParamValueFromAlias ( params , filters . Parameters , p . Name , p . Aliases )
}
2020-07-15 21:05:22 +02:00
for _ , s := range secrets {
params = setParamValueFromAlias ( params , filters . Parameters , s . Name , s . Aliases )
}
2019-10-29 11:58:24 +02:00
2020-04-01 20:46:33 +02:00
stepConfig . mixIn ( params , filters . Parameters )
2019-10-29 11:58:24 +02:00
}
2019-10-22 15:41:27 +02:00
}
2019-11-22 11:30:44 +02:00
// merge command line flags
2019-10-22 15:41:27 +02:00
if flagValues != nil {
stepConfig . mixIn ( flagValues , filters . Parameters )
}
2020-09-16 14:50:09 +02:00
stepConfig . mixIn ( c . General , vaultFilter )
2020-07-22 11:15:48 +02:00
// fetch secrets from vault
2020-09-16 14:50:09 +02:00
vaultClient , err := getVaultClientFromConfig ( stepConfig , c . vaultCredentials )
2020-07-22 11:15:48 +02:00
if err != nil {
return StepConfig { } , err
}
2020-09-16 14:50:09 +02:00
if vaultClient != nil {
err = addVaultCredentials ( & stepConfig , vaultClient , parameters )
if err != nil {
return StepConfig { } , err
}
2020-07-22 11:15:48 +02:00
}
2019-11-05 17:30:41 +02:00
// finally do the condition evaluation post processing
for _ , p := range parameters {
if len ( p . Conditions ) > 0 {
cp := p . Conditions [ 0 ] . Params [ 0 ]
dependentValue := stepConfig . Config [ cp . Name ]
if cmp . Equal ( dependentValue , cp . Value ) && stepConfig . Config [ p . Name ] == nil {
subMapValue := stepConfig . Config [ dependentValue . ( string ) ] . ( map [ string ] interface { } ) [ p . Name ]
if subMapValue != nil {
stepConfig . Config [ p . Name ] = subMapValue
} else {
stepConfig . Config [ p . Name ] = p . Default
}
}
}
}
2019-10-22 15:41:27 +02:00
return stepConfig , nil
}
2020-09-16 14:50:09 +02:00
// SetVaultCredentials sets the appRoleID and the appRoleSecretID to load additional configuration from vault
func ( c * Config ) SetVaultCredentials ( appRoleID , appRoleSecretID string ) {
c . vaultCredentials = VaultCredentials {
AppRoleID : appRoleID ,
AppRoleSecretID : appRoleSecretID ,
}
}
2019-10-22 15:41:27 +02:00
// GetStepConfigWithJSON provides merged step configuration using a provided stepConfigJSON with additional flags provided
func GetStepConfigWithJSON ( flagValues map [ string ] interface { } , stepConfigJSON string , filters StepFilters ) StepConfig {
var stepConfig StepConfig
stepConfigMap := map [ string ] interface { } { }
2020-04-01 20:46:33 +02:00
err := json . Unmarshal ( [ ] byte ( stepConfigJSON ) , & stepConfigMap )
if err != nil {
log . Entry ( ) . Warnf ( "invalid stepConfig JSON: %v" , err )
}
2019-10-22 15:41:27 +02:00
stepConfig . mixIn ( stepConfigMap , filters . All )
// ToDo: mix in parametersJSON
if flagValues != nil {
stepConfig . mixIn ( flagValues , filters . Parameters )
}
return stepConfig
}
// GetJSON returns JSON representation of an object
func GetJSON ( data interface { } ) ( string , error ) {
result , err := json . Marshal ( data )
if err != nil {
return "" , errors . Wrapf ( err , "error marshalling json: %v" , err )
}
return string ( result ) , nil
}
2019-11-21 17:09:57 +02:00
// OpenPiperFile provides functionality to retrieve configuration via file or http
func OpenPiperFile ( name string ) ( io . ReadCloser , error ) {
2020-03-31 08:47:09 +02:00
if ! strings . HasPrefix ( name , "http://" ) && ! strings . HasPrefix ( name , "https://" ) {
2019-11-21 17:09:57 +02:00
return os . Open ( name )
}
2020-03-31 08:47:09 +02:00
// support http(s) urls next to file path - url cannot be protected
client := http . Client { }
response , err := client . SendRequest ( "GET" , name , nil , nil , nil )
2020-04-01 20:46:33 +02:00
if err != nil {
return nil , err
}
return response . Body , nil
2019-11-21 17:09:57 +02:00
}
2019-10-22 15:41:27 +02:00
func envValues ( filter [ ] string ) map [ string ] interface { } {
vals := map [ string ] interface { } { }
for _ , param := range filter {
if envVal := os . Getenv ( "PIPER_" + param ) ; len ( envVal ) != 0 {
vals [ param ] = os . Getenv ( "PIPER_" + param )
}
}
return vals
}
func ( s * StepConfig ) mixIn ( mergeData map [ string ] interface { } , filter [ ] string ) {
if s . Config == nil {
s . Config = map [ string ] interface { } { }
}
2019-11-05 17:30:41 +02:00
s . Config = merge ( s . Config , filterMap ( mergeData , filter ) )
2019-10-22 15:41:27 +02:00
}
2019-11-22 11:30:44 +02:00
func ( s * StepConfig ) mixInStepDefaults ( stepParams [ ] StepParameters ) {
if s . Config == nil {
s . Config = map [ string ] interface { } { }
}
for _ , p := range stepParams {
if p . Default != nil {
s . Config [ p . Name ] = p . Default
}
}
}
2020-07-10 10:32:26 +02:00
// ApplyContainerConditions evaluates conditions in step yaml container definitions
func ApplyContainerConditions ( containers [ ] Container , stepConfig * StepConfig ) {
for _ , container := range containers {
if len ( container . Conditions ) > 0 {
for _ , param := range container . Conditions [ 0 ] . Params {
if container . Conditions [ 0 ] . ConditionRef == "strings-equal" && stepConfig . Config [ param . Name ] == param . Value {
var containerConf map [ string ] interface { }
if stepConfig . Config [ param . Value ] != nil {
containerConf = stepConfig . Config [ param . Value ] . ( map [ string ] interface { } )
for key , value := range containerConf {
if stepConfig . Config [ key ] == nil {
stepConfig . Config [ key ] = value
}
}
delete ( stepConfig . Config , param . Value )
}
}
}
}
}
}
2019-10-22 15:41:27 +02:00
func filterMap ( data map [ string ] interface { } , filter [ ] string ) map [ string ] interface { } {
result := map [ string ] interface { } { }
if data == nil {
data = map [ string ] interface { } { }
}
for key , value := range data {
if len ( filter ) == 0 || sliceContains ( filter , key ) {
result [ key ] = value
}
}
return result
}
func merge ( base , overlay map [ string ] interface { } ) map [ string ] interface { } {
result := map [ string ] interface { } { }
if base == nil {
base = map [ string ] interface { } { }
}
for key , value := range base {
result [ key ] = value
}
for key , value := range overlay {
if val , ok := value . ( map [ string ] interface { } ) ; ok {
if valBaseKey , ok := base [ key ] . ( map [ string ] interface { } ) ; ! ok {
result [ key ] = merge ( map [ string ] interface { } { } , val )
} else {
result [ key ] = merge ( valBaseKey , val )
}
} else {
result [ key ] = value
}
}
return result
}
func sliceContains ( slice [ ] string , find string ) bool {
for _ , elem := range slice {
if elem == find {
return true
}
}
return false
}