1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2024-12-14 11:03:09 +02:00
sap-jenkins-library/pkg/config/config.go
Eugene Kortelyov 56be54c504
Feature/vault refactoring (#3113)
* refactor vault code

* adjust generator

* wip: fix tests

* regenerate influxdb

* fix test

* add another test

* fix test & docs

* fix formatting

* Minorupdate and fixes

Co-authored-by: Kevin Stiehl <kevin.stiehl@numericas.de>
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-09-21 13:06:32 +02:00

482 lines
15 KiB
Go

package config
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"reflect"
"strings"
piperhttp "github.com/SAP/jenkins-library/pkg/http"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/ghodss/yaml"
"github.com/google/go-cmp/cmp"
"github.com/pkg/errors"
)
// Config defines the structure of the config files
type Config struct {
CustomDefaults []string `json:"customDefaults,omitempty"`
General map[string]interface{} `json:"general"`
Stages map[string]map[string]interface{} `json:"stages"`
Steps map[string]map[string]interface{} `json:"steps"`
Hooks map[string]interface{} `json:"hooks,omitempty"`
defaults PipelineDefaults
initialized bool
accessTokens map[string]string
openFile func(s string, t map[string]string) (io.ReadCloser, error)
vaultCredentials VaultCredentials
}
// StepConfig defines the structure for merged step configuration
type StepConfig struct {
Config map[string]interface{}
HookConfig map[string]interface{}
}
// ReadConfig loads config and returns its content
func (c *Config) ReadConfig(configuration io.ReadCloser) error {
defer configuration.Close()
content, err := ioutil.ReadAll(configuration)
if err != nil {
return errors.Wrapf(err, "error reading %v", configuration)
}
err = yaml.Unmarshal(content, &c)
if err != nil {
return NewParseError(fmt.Sprintf("format of configuration is invalid %q: %v", content, err))
}
return nil
}
// ApplyAliasConfig adds configuration values available on aliases to primary configuration parameters
func (c *Config) ApplyAliasConfig(parameters []StepParameters, secrets []StepSecrets, filters StepFilters, stageName, stepName string, stepAliases []Alias) {
// copy configuration from step alias to correct step
if len(stepAliases) > 0 {
c.copyStepAliasConfig(stepName, stepAliases)
}
for _, p := range parameters {
c.General = setParamValueFromAlias(stepName, c.General, filters.General, p.Name, p.Aliases)
if c.Stages[stageName] != nil {
c.Stages[stageName] = setParamValueFromAlias(stepName, c.Stages[stageName], filters.Stages, p.Name, p.Aliases)
}
if c.Steps[stepName] != nil {
c.Steps[stepName] = setParamValueFromAlias(stepName, c.Steps[stepName], filters.Steps, p.Name, p.Aliases)
}
}
for _, s := range secrets {
c.General = setParamValueFromAlias(stepName, c.General, filters.General, s.Name, s.Aliases)
if c.Stages[stageName] != nil {
c.Stages[stageName] = setParamValueFromAlias(stepName, c.Stages[stageName], filters.Stages, s.Name, s.Aliases)
}
if c.Steps[stepName] != nil {
c.Steps[stepName] = setParamValueFromAlias(stepName, c.Steps[stepName], filters.Steps, s.Name, s.Aliases)
}
}
}
func setParamValueFromAlias(stepName string, configMap map[string]interface{}, filter []string, name string, aliases []Alias) map[string]interface{} {
if configMap != nil && configMap[name] == nil && sliceContains(filter, name) {
for _, a := range aliases {
aliasVal := getDeepAliasValue(configMap, a.Name)
if aliasVal != nil {
configMap[name] = aliasVal
if a.Deprecated {
log.Entry().Warningf("[WARNING] The parameter '%v' is DEPRECATED, use '%v' instead. (%v/%v)", a.Name, name, log.LibraryName, stepName)
}
}
if configMap[name] != nil {
return configMap
}
}
}
return configMap
}
func getDeepAliasValue(configMap map[string]interface{}, key string) interface{} {
parts := strings.Split(key, "/")
if len(parts) > 1 {
if configMap[parts[0]] == nil {
return nil
}
paramValueType := reflect.ValueOf(configMap[parts[0]])
if paramValueType.Kind() != reflect.Map {
log.Entry().Debugf("Ignoring alias '%v' as '%v' is not pointing to a map.", key, parts[0])
return nil
}
return getDeepAliasValue(configMap[parts[0]].(map[string]interface{}), strings.Join(parts[1:], "/"))
}
return configMap[key]
}
func (c *Config) copyStepAliasConfig(stepName string, stepAliases []Alias) {
for _, stepAlias := range stepAliases {
if c.Steps[stepAlias.Name] != nil {
if stepAlias.Deprecated {
log.Entry().WithField("package", "SAP/jenkins-library/pkg/config").Warningf("DEPRECATION NOTICE: step configuration available for deprecated step '%v'. Please remove or move configuration to step '%v'!", stepAlias.Name, stepName)
}
for paramName, paramValue := range c.Steps[stepAlias.Name] {
if c.Steps[stepName] == nil {
c.Steps[stepName] = map[string]interface{}{}
}
if c.Steps[stepName][paramName] == nil {
c.Steps[stepName][paramName] = paramValue
}
}
}
}
}
// InitializeConfig prepares the config object, i.e. loading content, etc.
func (c *Config) InitializeConfig(configuration io.ReadCloser, defaults []io.ReadCloser, ignoreCustomDefaults bool) error {
if configuration != nil {
if err := c.ReadConfig(configuration); err != nil {
return errors.Wrap(err, "failed to parse custom pipeline configuration")
}
}
// consider custom defaults defined in config.yml unless told otherwise
if ignoreCustomDefaults {
log.Entry().Info("Ignoring custom defaults from pipeline config")
} else if c.CustomDefaults != nil && len(c.CustomDefaults) > 0 {
if c.openFile == nil {
c.openFile = OpenPiperFile
}
for _, f := range c.CustomDefaults {
fc, err := c.openFile(f, c.accessTokens)
if err != nil {
return errors.Wrapf(err, "getting default '%v' failed", f)
}
defaults = append(defaults, fc)
}
}
if err := c.defaults.ReadPipelineDefaults(defaults); err != nil {
return errors.Wrap(err, "failed to read default configuration")
}
c.initialized = true
return nil
}
// GetStepConfig provides merged step configuration using defaults, config, if available
func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON string, configuration io.ReadCloser, defaults []io.ReadCloser, ignoreCustomDefaults bool, filters StepFilters, parameters []StepParameters, secrets []StepSecrets, envParameters map[string]interface{}, stageName, stepName string, stepAliases []Alias) (StepConfig, error) {
var stepConfig StepConfig
var err error
if !c.initialized {
err = c.InitializeConfig(configuration, defaults, ignoreCustomDefaults)
if err != nil {
return StepConfig{}, err
}
}
c.ApplyAliasConfig(parameters, secrets, filters, stageName, stepName, stepAliases)
// initialize with defaults from step.yaml
stepConfig.mixInStepDefaults(parameters)
// merge parameters provided by Piper environment
stepConfig.mixIn(envParameters, filters.All)
// read defaults & merge general -> steps (-> general -> steps ...)
for _, def := range c.defaults.Defaults {
def.ApplyAliasConfig(parameters, secrets, filters, stageName, stepName, stepAliases)
stepConfig.mixIn(def.General, filters.General)
stepConfig.mixIn(def.Steps[stepName], filters.Steps)
stepConfig.mixIn(def.Stages[stageName], filters.Steps)
stepConfig.mixinVaultConfig(parameters, def.General, def.Steps[stepName], def.Stages[stageName])
stepConfig.mixInHookConfig(def.Hooks)
}
// read config & merge - general -> steps -> stages
stepConfig.mixIn(c.General, filters.General)
stepConfig.mixIn(c.Steps[stepName], filters.Steps)
stepConfig.mixIn(c.Stages[stageName], filters.Stages)
// merge parameters provided via env vars
stepConfig.mixIn(envValues(filters.All), filters.All)
// if parameters are provided in JSON format merge them
if len(paramJSON) != 0 {
var params map[string]interface{}
err := json.Unmarshal([]byte(paramJSON), &params)
if err != nil {
log.Entry().Warnf("failed to parse parameters from environment: %v", err)
} else {
//apply aliases
for _, p := range parameters {
params = setParamValueFromAlias(stepName, params, filters.Parameters, p.Name, p.Aliases)
}
for _, s := range secrets {
params = setParamValueFromAlias(stepName, params, filters.Parameters, s.Name, s.Aliases)
}
stepConfig.mixIn(params, filters.Parameters)
}
}
// merge command line flags
if flagValues != nil {
stepConfig.mixIn(flagValues, filters.Parameters)
}
if verbose, ok := stepConfig.Config["verbose"].(bool); ok && verbose {
log.SetVerbose(verbose)
} else if !ok && stepConfig.Config["verbose"] != nil {
log.Entry().Warnf("invalid value for parameter verbose: '%v'", stepConfig.Config["verbose"])
}
stepConfig.mixinVaultConfig(parameters, c.General, c.Steps[stepName], c.Stages[stageName])
// check whether vault should be skipped
if skip, ok := stepConfig.Config["skipVault"].(bool); !ok || !skip {
// fetch secrets from vault
vaultClient, err := getVaultClientFromConfig(stepConfig, c.vaultCredentials)
if err != nil {
return StepConfig{}, err
}
if vaultClient != nil {
defer vaultClient.MustRevokeToken()
resolveAllVaultReferences(&stepConfig, vaultClient, parameters)
resolveVaultTestCredentials(&stepConfig, vaultClient)
}
}
// finally do the condition evaluation post processing
for _, p := range parameters {
if len(p.Conditions) > 0 {
for _, cond := range p.Conditions {
for _, param := range cond.Params {
// retrieve configuration value of condition parameter
dependentValue := stepConfig.Config[param.Name]
// check if configuration of condition parameter matches the value
// so far string-equals condition is assumed here
// if so and if no config applied yet, then try to apply the value
if cmp.Equal(dependentValue, param.Value) && stepConfig.Config[p.Name] == nil {
subMap, ok := stepConfig.Config[dependentValue.(string)].(map[string]interface{})
if ok && subMap[p.Name] != nil {
stepConfig.Config[p.Name] = subMap[p.Name]
}
}
}
}
}
}
return stepConfig, nil
}
// SetVaultCredentials sets the appRoleID and the appRoleSecretID or the vaultTokento load additional
//configuration from vault
// Either appRoleID and appRoleSecretID or vaultToken must be specified.
func (c *Config) SetVaultCredentials(appRoleID, appRoleSecretID string, vaultToken string) {
c.vaultCredentials = VaultCredentials{
AppRoleID: appRoleID,
AppRoleSecretID: appRoleSecretID,
VaultToken: vaultToken,
}
}
// GetStepConfigWithJSON provides merged step configuration using a provided stepConfigJSON with additional flags provided
func GetStepConfigWithJSON(flagValues map[string]interface{}, stepConfigJSON string, filters StepFilters) StepConfig {
var stepConfig StepConfig
stepConfigMap := map[string]interface{}{}
err := json.Unmarshal([]byte(stepConfigJSON), &stepConfigMap)
if err != nil {
log.Entry().Warnf("invalid stepConfig JSON: %v", err)
}
stepConfig.mixIn(stepConfigMap, filters.All)
// ToDo: mix in parametersJSON
if flagValues != nil {
stepConfig.mixIn(flagValues, filters.Parameters)
}
return stepConfig
}
func (c *Config) GetStageConfig(paramJSON string, configuration io.ReadCloser, defaults []io.ReadCloser, ignoreCustomDefaults bool, acceptedParams []string, stageName string) (StepConfig, error) {
filters := StepFilters{
General: acceptedParams,
Steps: []string{},
Stages: acceptedParams,
Parameters: acceptedParams,
Env: []string{},
}
return c.GetStepConfig(map[string]interface{}{}, paramJSON, configuration, defaults, ignoreCustomDefaults, filters, []StepParameters{}, []StepSecrets{}, map[string]interface{}{}, stageName, "", []Alias{})
}
// GetJSON returns JSON representation of an object
func GetJSON(data interface{}) (string, error) {
result, err := json.Marshal(data)
if err != nil {
return "", errors.Wrapf(err, "error marshalling json: %v", err)
}
return string(result), nil
}
// OpenPiperFile provides functionality to retrieve configuration via file or http
func OpenPiperFile(name string, accessTokens map[string]string) (io.ReadCloser, error) {
if !strings.HasPrefix(name, "http://") && !strings.HasPrefix(name, "https://") {
return os.Open(name)
}
return httpReadFile(name, accessTokens)
}
func httpReadFile(name string, accessTokens map[string]string) (io.ReadCloser, error) {
u, err := url.Parse(name)
if err != nil {
return nil, fmt.Errorf("failed to read url: %w", err)
}
// support http(s) urls next to file path
client := piperhttp.Client{}
var header http.Header
if len(accessTokens[u.Host]) > 0 {
client.SetOptions(piperhttp.ClientOptions{Token: fmt.Sprintf("token %v", accessTokens[u.Host])})
header = map[string][]string{"Accept": {"application/vnd.github.v3.raw"}}
}
response, err := client.SendRequest("GET", name, nil, header, nil)
if err != nil {
return nil, err
}
return response.Body, nil
}
func envValues(filter []string) map[string]interface{} {
vals := map[string]interface{}{}
for _, param := range filter {
if envVal := os.Getenv("PIPER_" + param); len(envVal) != 0 {
vals[param] = os.Getenv("PIPER_" + param)
}
}
return vals
}
func (s *StepConfig) mixIn(mergeData map[string]interface{}, filter []string) {
if s.Config == nil {
s.Config = map[string]interface{}{}
}
s.Config = merge(s.Config, filterMap(mergeData, filter))
}
func (s *StepConfig) mixInHookConfig(mergeData map[string]interface{}) {
if s.HookConfig == nil {
s.HookConfig = map[string]interface{}{}
}
s.HookConfig = merge(s.HookConfig, mergeData)
}
func (s *StepConfig) mixInStepDefaults(stepParams []StepParameters) {
if s.Config == nil {
s.Config = map[string]interface{}{}
}
// conditional defaults need to be written to a sub map
// in order to prevent a "last one wins" situation
// this is then considered at the end of GetStepConfig once the complete configuration is known
for _, p := range stepParams {
if p.Default != nil {
if len(p.Conditions) == 0 {
s.Config[p.Name] = p.Default
} else {
for _, cond := range p.Conditions {
for _, param := range cond.Params {
s.Config[param.Value] = map[string]interface{}{p.Name: p.Default}
}
}
}
}
}
}
// ApplyContainerConditions evaluates conditions in step yaml container definitions
func ApplyContainerConditions(containers []Container, stepConfig *StepConfig) {
for _, container := range containers {
if len(container.Conditions) > 0 {
for _, param := range container.Conditions[0].Params {
if container.Conditions[0].ConditionRef == "strings-equal" && stepConfig.Config[param.Name] == param.Value {
var containerConf map[string]interface{}
if stepConfig.Config[param.Value] != nil {
containerConf = stepConfig.Config[param.Value].(map[string]interface{})
for key, value := range containerConf {
if stepConfig.Config[key] == nil {
stepConfig.Config[key] = value
}
}
delete(stepConfig.Config, param.Value)
}
}
}
}
}
}
func filterMap(data map[string]interface{}, filter []string) map[string]interface{} {
result := map[string]interface{}{}
if data == nil {
data = map[string]interface{}{}
}
for key, value := range data {
if value != nil && (len(filter) == 0 || sliceContains(filter, key)) {
result[key] = value
}
}
return result
}
func merge(base, overlay map[string]interface{}) map[string]interface{} {
result := map[string]interface{}{}
if base == nil {
base = map[string]interface{}{}
}
for key, value := range base {
result[key] = value
}
for key, value := range overlay {
if val, ok := value.(map[string]interface{}); ok {
if valBaseKey, ok := base[key].(map[string]interface{}); !ok {
result[key] = merge(map[string]interface{}{}, val)
} else {
result[key] = merge(valBaseKey, val)
}
} else {
result[key] = value
}
}
return result
}
func sliceContains(slice []string, find string) bool {
for _, elem := range slice {
if elem == find {
return true
}
}
return false
}