From 3eae0c5f68d8925f505ae0f904f311c824b576d9 Mon Sep 17 00:00:00 2001 From: Kevin Stiehl Date: Tue, 13 Oct 2020 14:14:47 +0200 Subject: [PATCH] feat(vault): fetch secrets from vault (#2032) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cloud-foundry & sonar from vault * add vault development hint * don't abort on vault errors * cloudfoundry make credentialsId only mandatory when vault is not configured * add vault ref to step ymls * rename vaultAddress to vaultServerUrl * rename PIPER_vaultRole* to PIPER_vaultAppRole* * add resourceRef for detect step * fix error when no namespace is set * added debug logs * added debug logs * fix vault resolving * add vaultCustomBasePath * rename vault_test.go to client_test.go * refactored vault logging * refactored config param lookup for vault * added tüddelchen * rename vaultCustomBasePath to vaultPath * fix tests * change lookup path for group secrets * fix interpolation tests * added vault resource ref to versioning * execute go generate * rename Approle to AppRole * change verbose back to false Co-authored-by: Leander Schulz Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> --- DEVELOPMENT.md | 4 + cmd/artifactPrepareVersion_generated.go | 12 +++ cmd/checkmarxExecuteScan_generated.go | 12 +++ cmd/cloudFoundryCreateServiceKey_generated.go | 12 +++ cmd/cloudFoundryCreateService_generated.go | 12 +++ cmd/cloudFoundryDeleteService_generated.go | 12 +++ cmd/cloudFoundryDeploy_generated.go | 12 +++ cmd/detectExecuteScan_generated.go | 6 ++ cmd/fortifyExecuteScan_generated.go | 6 ++ cmd/piper.go | 4 +- cmd/sonarExecuteScan_generated.go | 6 ++ .../developer_hints/VaultResourceReference.md | 29 +++++++ pkg/config/config.go | 5 +- pkg/config/interpolation/interpolation.go | 24 +++--- .../interpolation/interpolation_test.go | 21 ++++-- pkg/config/stepmeta.go | 2 +- pkg/config/stepmeta_test.go | 12 +-- pkg/config/vault.go | 75 +++++++++++++------ pkg/config/vault_test.go | 41 ++++++---- pkg/vault/{vault.go => client.go} | 5 +- pkg/vault/{vault_test.go => client_test.go} | 0 resources/metadata/checkmarx.yaml | 10 +++ .../metadata/cloudFoundryCreateService.yaml | 10 +++ .../cloudFoundryCreateServiceKey.yaml | 10 +++ .../metadata/cloudFoundryDeleteService.yaml | 10 +++ resources/metadata/cloudFoundryDeploy.yaml | 10 +++ resources/metadata/detect.yaml | 5 ++ resources/metadata/fortify.yaml | 5 ++ resources/metadata/sonar.yaml | 5 ++ resources/metadata/versioning.yaml | 10 +++ vars/cloudFoundryDeploy.groovy | 2 +- vars/piperExecuteBin.groovy | 4 + 32 files changed, 323 insertions(+), 70 deletions(-) create mode 100644 documentation/developer_hints/VaultResourceReference.md rename pkg/vault/{vault.go => client.go} (94%) rename pkg/vault/{vault_test.go => client_test.go} (100%) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c068190ff..e50e2431c 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -481,3 +481,7 @@ On initialization, it merges the provided custom default configurations with the Note, the list of configurations cached by `DefaultValueCache` is used to pass path to the (custom) default configurations of each Go step. It only contains the paths of configurations which are **not** provided via `customDefaults` parameter of the project configuration, since the Go layer already resolves configurations provided via `customDefaults` parameter independently. + +## Additional Developer Hints + +You can find additional hints at [documentation/developer-hints](./documentation/developer_hints) diff --git a/cmd/artifactPrepareVersion_generated.go b/cmd/artifactPrepareVersion_generated.go index 81c91937f..a43ec02e1 100644 --- a/cmd/artifactPrepareVersion_generated.go +++ b/cmd/artifactPrepareVersion_generated.go @@ -312,6 +312,12 @@ func artifactPrepareVersionMetadata() config.StepData { Param: "password", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/versioning", "$(vaultBasePath)/$(vaultPipelineName)/versioning", "$(vaultBasePath)/GROUP-SECRETS/versioning"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", @@ -358,6 +364,12 @@ func artifactPrepareVersionMetadata() config.StepData { Param: "username", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/versioning", "$(vaultBasePath)/$(vaultPipelineName)/versioning", "$(vaultBasePath)/GROUP-SECRETS/versioning"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", diff --git a/cmd/checkmarxExecuteScan_generated.go b/cmd/checkmarxExecuteScan_generated.go index 1d20781c1..0eee7e5e4 100644 --- a/cmd/checkmarxExecuteScan_generated.go +++ b/cmd/checkmarxExecuteScan_generated.go @@ -323,6 +323,12 @@ func checkmarxExecuteScanMetadata() config.StepData { Param: "password", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/checkmarx", "$(vaultBasePath)/$(vaultPipelineName)/checkmarx", "$(vaultBasePath)/GROUP-SECRETS/checkmarx"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", @@ -393,6 +399,12 @@ func checkmarxExecuteScanMetadata() config.StepData { Param: "username", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/checkmarx", "$(vaultBasePath)/$(vaultPipelineName)/checkmarx", "$(vaultBasePath)/GROUP-SECRETS/checkmarx"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", diff --git a/cmd/cloudFoundryCreateServiceKey_generated.go b/cmd/cloudFoundryCreateServiceKey_generated.go index 872aa5b1b..ab2b366e2 100644 --- a/cmd/cloudFoundryCreateServiceKey_generated.go +++ b/cmd/cloudFoundryCreateServiceKey_generated.go @@ -126,6 +126,12 @@ func cloudFoundryCreateServiceKeyMetadata() config.StepData { Param: "username", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace)"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", @@ -140,6 +146,12 @@ func cloudFoundryCreateServiceKeyMetadata() config.StepData { Param: "password", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace)"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", diff --git a/cmd/cloudFoundryCreateService_generated.go b/cmd/cloudFoundryCreateService_generated.go index e1a5e957a..26b0cda58 100644 --- a/cmd/cloudFoundryCreateService_generated.go +++ b/cmd/cloudFoundryCreateService_generated.go @@ -142,6 +142,12 @@ func cloudFoundryCreateServiceMetadata() config.StepData { Param: "username", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace)"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", @@ -156,6 +162,12 @@ func cloudFoundryCreateServiceMetadata() config.StepData { Param: "password", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace)"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", diff --git a/cmd/cloudFoundryDeleteService_generated.go b/cmd/cloudFoundryDeleteService_generated.go index 42665bd2d..f65daffdf 100644 --- a/cmd/cloudFoundryDeleteService_generated.go +++ b/cmd/cloudFoundryDeleteService_generated.go @@ -123,6 +123,12 @@ func cloudFoundryDeleteServiceMetadata() config.StepData { Param: "username", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace)"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", @@ -137,6 +143,12 @@ func cloudFoundryDeleteServiceMetadata() config.StepData { Param: "password", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace)"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", diff --git a/cmd/cloudFoundryDeploy_generated.go b/cmd/cloudFoundryDeploy_generated.go index 29eec06b5..9f49e0da5 100644 --- a/cmd/cloudFoundryDeploy_generated.go +++ b/cmd/cloudFoundryDeploy_generated.go @@ -402,6 +402,12 @@ func cloudFoundryDeployMetadata() config.StepData { Param: "password", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace)"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", @@ -440,6 +446,12 @@ func cloudFoundryDeployMetadata() config.StepData { Param: "username", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace)", "$(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace)"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", diff --git a/cmd/detectExecuteScan_generated.go b/cmd/detectExecuteScan_generated.go index 546d8831a..f7aa9a88b 100644 --- a/cmd/detectExecuteScan_generated.go +++ b/cmd/detectExecuteScan_generated.go @@ -126,6 +126,12 @@ func detectExecuteScanMetadata() config.StepData { Name: "detectTokenCredentialsId", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/detect", "$(vaultBasePath)/$(vaultPipelineName)/detect", "$(vaultBasePath)/GROUP-SECRETS/detect"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", diff --git a/cmd/fortifyExecuteScan_generated.go b/cmd/fortifyExecuteScan_generated.go index 862dcaf5a..f87bcf7c4 100644 --- a/cmd/fortifyExecuteScan_generated.go +++ b/cmd/fortifyExecuteScan_generated.go @@ -260,6 +260,12 @@ func fortifyExecuteScanMetadata() config.StepData { Name: "fortifyCredentialsId", Type: "secret", }, + + { + Name: "", + Paths: []string{"$(vaultPath)/fortify", "$(vaultBasePath)/$(vaultPipelineName)/fortify", "$(vaultBasePath)/GROUP-SECRETS/fortify"}, + Type: "vaultSecret", + }, }, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", diff --git a/cmd/piper.go b/cmd/piper.go index d0d20d05d..877cfe4f4 100644 --- a/cmd/piper.go +++ b/cmd/piper.go @@ -207,10 +207,10 @@ func PrepareConfig(cmd *cobra.Command, metadata *config.StepData, stepName strin // add vault credentials so that configuration can be fetched from vault if GeneralConfig.VaultRoleID == "" { - GeneralConfig.VaultRoleID = os.Getenv("PIPER_vaultRoleID") + GeneralConfig.VaultRoleID = os.Getenv("PIPER_vaultAppRoleID") } if GeneralConfig.VaultRoleSecretID == "" { - GeneralConfig.VaultRoleSecretID = os.Getenv("PIPER_vaultRoleSecretID") + GeneralConfig.VaultRoleSecretID = os.Getenv("PIPER_vaultAppRoleSecretID") } myConfig.SetVaultCredentials(GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID) diff --git a/cmd/sonarExecuteScan_generated.go b/cmd/sonarExecuteScan_generated.go index a3e411110..031eb91cb 100644 --- a/cmd/sonarExecuteScan_generated.go +++ b/cmd/sonarExecuteScan_generated.go @@ -193,6 +193,12 @@ func sonarExecuteScanMetadata() config.StepData { { Name: "token", ResourceRef: []config.ResourceReference{ + { + Name: "", + Paths: []string{"$(vaultPath)/sonar", "$(vaultBasePath)/$(vaultPipelineName)/sonar", "$(vaultBasePath)/GROUP-SECRETS/sonar"}, + Type: "vaultSecret", + }, + { Name: "sonarTokenCredentialsId", Type: "secret", diff --git a/documentation/developer_hints/VaultResourceReference.md b/documentation/developer_hints/VaultResourceReference.md new file mode 100644 index 000000000..0b32405de --- /dev/null +++ b/documentation/developer_hints/VaultResourceReference.md @@ -0,0 +1,29 @@ +# The Vault ResourceRef + +## Preconditions + +Parameters that have a ResourceReference of type `vaultSecret` will be looked up from vault when all of the following things are true... + +* The environment variables `PIPER_vaultAppRoleID` and `PIPER_vaultAppRoleSecretID` must both be set to the Vault AppRole role ID and to the Vault AppRole secret ID. See [Vault AppRole docs](https://www.vaultproject.io/docs/auth/approle) +* `vaultServerUrl` ist set in the `general` section of the configuration file. +* The parameter must not be set by the configuration file, as a CLI Parameter or an environment variable. Any parameter that has already been set won't be resolved via vault. + +## Lookup + +``` +- name: token + type: string + description: "Token used to authenticate with the Sonar Server." + scope: + - PARAMETERS + secret: true + resourceRef: + - type: vaultSecret + paths: + - $(vaultBasePath)/$(vaultPipelineName)/sonar + - $(vaultBasePath)/__group/sonar +``` + +With the example above piper will check whether the the `token` parameter has already been set when the config was resolved. If `token` hasn't be resolved yet we will go through every item of the `paths` array, interpolate every string by using the already resolved config and then check whether there is a secret stored at the given path. + +In case we find a secret we check whether it has a field (secrets in vault are **flat** json documents) that matches the parameters name (or one of the alias names), in the example above this would be `token`. diff --git a/pkg/config/config.go b/pkg/config/config.go index 643ca47b5..53be22440 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -234,10 +234,7 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri return StepConfig{}, err } if vaultClient != nil { - err = addVaultCredentials(&stepConfig, vaultClient, parameters) - if err != nil { - return StepConfig{}, err - } + addVaultCredentials(&stepConfig, vaultClient, parameters) } // finally do the condition evaluation post processing diff --git a/pkg/config/interpolation/interpolation.go b/pkg/config/interpolation/interpolation.go index e69449a4f..53922a9b6 100644 --- a/pkg/config/interpolation/interpolation.go +++ b/pkg/config/interpolation/interpolation.go @@ -4,6 +4,8 @@ import ( "fmt" "regexp" "strings" + + "github.com/SAP/jenkins-library/pkg/log" ) const ( @@ -16,33 +18,35 @@ var ( ) // ResolveMap interpolates every string value of a map and tries to lookup references to other properties of that map -func ResolveMap(config map[string]interface{}) error { +func ResolveMap(config map[string]interface{}) bool { for key, value := range config { if str, ok := value.(string); ok { - resolvedStr, err := ResolveString(str, config) - if err != nil { - return err + resolvedStr, ok := ResolveString(str, config) + if !ok { + return false } config[key] = resolvedStr } } - return nil + return true } -func resolveString(str string, lookupMap map[string]interface{}, n int) (string, error) { +func resolveString(str string, lookupMap map[string]interface{}, n int) (string, bool) { matches := lookupRegex.FindAllStringSubmatch(str, -1) if len(matches) == 0 { - return str, nil + return str, true } if n == maxLookupDepth { - return "", fmt.Errorf("Property could not be resolved with a depth of %d. '%s' is still left to resolve", n, str) + log.Entry().Errorf("Property could not be resolved with a depth of %d. '%s' is still left to resolve", n, str) + return "", false } for _, match := range matches { property := match[captureGroups["property"]] if propVal, ok := lookupMap[property]; ok { str = strings.ReplaceAll(str, fmt.Sprintf("$(%s)", property), propVal.(string)) } else { - str = strings.ReplaceAll(str, fmt.Sprintf("$(%s)", property), "") + // value not found + return "", false } } return resolveString(str, lookupMap, n+1) @@ -50,7 +54,7 @@ func resolveString(str string, lookupMap map[string]interface{}, n int) (string, // ResolveString takes a string and replaces all references inside of it with values from the given lookupMap. // This is being done recursively until the maxLookupDepth is reached. -func ResolveString(str string, lookupMap map[string]interface{}) (string, error) { +func ResolveString(str string, lookupMap map[string]interface{}) (string, bool) { return resolveString(str, lookupMap, 0) } diff --git a/pkg/config/interpolation/interpolation_test.go b/pkg/config/interpolation/interpolation_test.go index b7922dfa8..0ae72e6be 100644 --- a/pkg/config/interpolation/interpolation_test.go +++ b/pkg/config/interpolation/interpolation_test.go @@ -9,26 +9,37 @@ import ( func TestResolveMap(t *testing.T) { t.Parallel() - t.Run("Lookup lookup works", func(t *testing.T) { + t.Run("That lookup works", func(t *testing.T) { testMap := map[string]interface{}{ "prop1": "val1", "prop2": "val2", "prop3": "$(prop1)/$(prop2)", } - err := ResolveMap(testMap) - assert.NoError(t, err) + ok := ResolveMap(testMap) + assert.True(t, ok) assert.Equal(t, "val1/val2", testMap["prop3"]) }) + t.Run("That lookups fails when property is not found", func(t *testing.T) { + testMap := map[string]interface{}{ + "prop1": "val1", + "prop2": "val2", + "prop3": "$(prop1)/$(prop2)/$(prop5)", + } + + ok := ResolveMap(testMap) + assert.False(t, ok) + }) + t.Run("That resolve loops are aborted", func(t *testing.T) { testMap := map[string]interface{}{ "prop1": "$(prop2)", "prop2": "$(prop1)", } - err := ResolveMap(testMap) - assert.Error(t, err) + ok := ResolveMap(testMap) + assert.False(t, ok) }) } diff --git a/pkg/config/stepmeta.go b/pkg/config/stepmeta.go index bc6fd15fd..c18830771 100644 --- a/pkg/config/stepmeta.go +++ b/pkg/config/stepmeta.go @@ -233,7 +233,7 @@ func (m *StepData) GetContextParameterFilters() StepFilters { } if m.HasReference("vaultSecret") { - contextFilters = append(contextFilters, []string{"vaultAppRoleCredentialId", "vaultAppRoleSecretCredentialId"}...) + contextFilters = append(contextFilters, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}...) } if len(contextFilters) > 0 { diff --git a/pkg/config/stepmeta_test.go b/pkg/config/stepmeta_test.go index e3a705969..6f6cd09f5 100644 --- a/pkg/config/stepmeta_test.go +++ b/pkg/config/stepmeta_test.go @@ -300,12 +300,12 @@ func TestGetContextParameterFilters(t *testing.T) { t.Run("Vault", func(t *testing.T) { filters := metadata4.GetContextParameterFilters() - assert.Equal(t, []string{"vaultAppRoleCredentialId", "vaultAppRoleSecretCredentialId"}, filters.All, "incorrect filter All") - assert.Equal(t, []string{"vaultAppRoleCredentialId", "vaultAppRoleSecretCredentialId"}, filters.General, "incorrect filter General") - assert.Equal(t, []string{"vaultAppRoleCredentialId", "vaultAppRoleSecretCredentialId"}, filters.Steps, "incorrect filter Steps") - assert.Equal(t, []string{"vaultAppRoleCredentialId", "vaultAppRoleSecretCredentialId"}, filters.Stages, "incorrect filter Stages") - assert.Equal(t, []string{"vaultAppRoleCredentialId", "vaultAppRoleSecretCredentialId"}, filters.Parameters, "incorrect filter Parameters") - assert.Equal(t, []string{"vaultAppRoleCredentialId", "vaultAppRoleSecretCredentialId"}, filters.Env, "incorrect filter Env") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.All, "incorrect filter All") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.General, "incorrect filter General") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Steps, "incorrect filter Steps") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Stages, "incorrect filter Stages") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Parameters, "incorrect filter Parameters") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Env, "incorrect filter Env") }) } diff --git a/pkg/config/vault.go b/pkg/config/vault.go index 5614ed879..6a6a25012 100644 --- a/pkg/config/vault.go +++ b/pkg/config/vault.go @@ -8,12 +8,13 @@ import ( ) var vaultFilter = []string{ - "vaultApproleID", - "vaultApproleSecreId", - "vaultAddress", + "vaultAppRoleID", + "vaultAppRoleSecreId", + "vaultServerUrl", "vaultNamespace", "vaultBasePath", "vaultPipelineName", + "vaultPath", } // VaultCredentials hold all the auth information needed to fetch configuration from vault @@ -28,16 +29,18 @@ type vaultClient interface { } func getVaultClientFromConfig(config StepConfig, creds VaultCredentials) (vaultClient, error) { - address, addressOk := config.Config["vaultAddress"].(string) - log.Entry().Infof("config received %#v", config.Config) + address, addressOk := config.Config["vaultServerUrl"].(string) // if vault isn't used it's not an error if !addressOk || creds.AppRoleID == "" || creds.AppRoleSecretID == "" { log.Entry().Info("Skipping fetching secrets from vault since it is not configured") return nil, nil } - + namespace := "" // namespaces are only available in vault enterprise so using them should be optional - namespace := config.Config["vaultNamespace"].(string) + if config.Config["vaultNamespace"] != nil { + namespace = config.Config["vaultNamespace"].(string) + log.Entry().Debugf("Using vault namespace %s", namespace) + } client, err := vault.NewClientWithAppRole(&api.Config{Address: address}, creds.AppRoleID, creds.AppRoleSecretID, namespace) if err != nil { @@ -48,9 +51,8 @@ func getVaultClientFromConfig(config StepConfig, creds VaultCredentials) (vaultC return &client, nil } -func addVaultCredentials(config *StepConfig, client vaultClient, params []StepParameters) error { +func addVaultCredentials(config *StepConfig, client vaultClient, params []StepParameters) { for _, param := range params { - // we don't overwrite secrets that have already been set in any way if _, ok := config.Config[param.Name].(string); ok { continue @@ -59,29 +61,54 @@ func addVaultCredentials(config *StepConfig, client vaultClient, params []StepPa if ref == nil { continue } + var secretValue *string for _, vaultPath := range ref.Paths { // it should be possible to configure the root path were the secret is stored - var err error - vaultPath, err = interpolation.ResolveString(vaultPath, config.Config) - if err != nil { - return err - } - - secret, err := client.GetKvSecret(vaultPath) - if err != nil { - return err - } - if secret == nil { + vaultPath, ok := interpolation.ResolveString(vaultPath, config.Config) + if !ok { continue } - field := secret[param.Name] - if field != "" { - log.RegisterSecret(field) - config.Config[param.Name] = field + secretValue = lookupPath(client, vaultPath, ¶m) + if secretValue != nil { + config.Config[param.Name] = *secretValue + log.Entry().Infof("Resolved param '%s' with vault path '%s'", param.Name, vaultPath) break } } + if secretValue == nil { + log.Entry().Warnf("Could not resolve param '%s' from vault", param.Name) + } + } +} + +func lookupPath(client vaultClient, path string, param *StepParameters) *string { + log.Entry().Infof("Trying to resolve vault parameter '%s' at '%s'", param.Name, path) + secret, err := client.GetKvSecret(path) + if err != nil { + log.Entry().WithError(err).Warnf("Couldn't fetch secret at '%s'", path) + return nil + } + if secret == nil { + return nil + } + + field := secret[param.Name] + if field != "" { + log.RegisterSecret(field) + return &field + } + + // try parameter aliases + for _, alias := range param.Aliases { + field := secret[param.Name] + if field != "" { + log.RegisterSecret(field) + if alias.Deprecated { + log.Entry().WithField("package", "SAP/jenkins-library/pkg/config").Warningf("DEPRECATION NOTICE: old step config key '%s' used in vault. Please switch to '%s'!", alias.Name, param.Name) + } + return &field + } } return nil } diff --git a/pkg/config/vault_test.go b/pkg/config/vault_test.go index 0a8665247..34c22fd59 100644 --- a/pkg/config/vault_test.go +++ b/pkg/config/vault_test.go @@ -4,6 +4,8 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/mock" + "github.com/SAP/jenkins-library/pkg/config/mocks" "github.com/stretchr/testify/assert" ) @@ -20,8 +22,7 @@ func TestVaultConfigLoad(t *testing.T) { vaultData := map[string]string{secretName: "value1"} vaultMock.On("GetKvSecret", "team1/pipelineA").Return(vaultData, nil) - err := addVaultCredentials(&stepConfig, vaultMock, stepParams) - assert.NoError(t, err) + addVaultCredentials(&stepConfig, vaultMock, stepParams) assert.Equal(t, "value1", stepConfig.Config[secretName]) }) @@ -34,9 +35,8 @@ func TestVaultConfigLoad(t *testing.T) { stepParams := []StepParameters{stepParam(secretName, "vaultSecret", "$(vaultBasePath)/pipelineA")} vaultData := map[string]string{secretName: "value1"} vaultMock.On("GetKvSecret", "team1/pipelineA").Return(vaultData, nil) - err := addVaultCredentials(&stepConfig, vaultMock, stepParams) + addVaultCredentials(&stepConfig, vaultMock, stepParams) - assert.NoError(t, err) assert.Equal(t, "preset value", stepConfig.Config[secretName]) }) @@ -47,9 +47,8 @@ func TestVaultConfigLoad(t *testing.T) { }} stepParams := []StepParameters{stepParam(secretName, "vaultSecret", "$(vaultBasePath)/pipelineA")} vaultMock.On("GetKvSecret", "team1/pipelineA").Return(nil, fmt.Errorf("test")) - err := addVaultCredentials(&stepConfig, vaultMock, stepParams) + addVaultCredentials(&stepConfig, vaultMock, stepParams) assert.Len(t, stepConfig.Config, 1) - assert.EqualError(t, err, "test") }) t.Run("Secret doesn't exist", func(t *testing.T) { @@ -59,8 +58,7 @@ func TestVaultConfigLoad(t *testing.T) { }} stepParams := []StepParameters{stepParam(secretName, "vaultSecret", "$(vaultBasePath)/pipelineA")} vaultMock.On("GetKvSecret", "team1/pipelineA").Return(nil, nil) - err := addVaultCredentials(&stepConfig, vaultMock, stepParams) - assert.NoError(t, err) + addVaultCredentials(&stepConfig, vaultMock, stepParams) assert.Len(t, stepConfig.Config, 1) }) @@ -75,22 +73,33 @@ func TestVaultConfigLoad(t *testing.T) { vaultData := map[string]string{secretName: "value1"} vaultMock.On("GetKvSecret", "team1/pipelineA").Return(nil, nil) vaultMock.On("GetKvSecret", "team1/pipelineB").Return(vaultData, nil) - err := addVaultCredentials(&stepConfig, vaultMock, stepParams) - assert.NoError(t, err) + addVaultCredentials(&stepConfig, vaultMock, stepParams) assert.Equal(t, "value1", stepConfig.Config[secretName]) }) + t.Run("Stop lookup when secret was found", func(t *testing.T) { + vaultMock := &mocks.VaultMock{} + stepConfig := StepConfig{Config: map[string]interface{}{ + "vaultBasePath": "team1", + }} + stepParams := []StepParameters{ + stepParam(secretName, "vaultSecret", "$(vaultBasePath)/pipelineA", "$(vaultBasePath)/pipelineB"), + } + vaultData := map[string]string{secretName: "value1"} + vaultMock.On("GetKvSecret", "team1/pipelineA").Return(vaultData, nil) + addVaultCredentials(&stepConfig, vaultMock, stepParams) + assert.Equal(t, "value1", stepConfig.Config[secretName]) + vaultMock.AssertNotCalled(t, "GetKvSecret", "team1/pipelineB") + }) + t.Run("No BasePath is stepConfig.Configured", func(t *testing.T) { vaultMock := &mocks.VaultMock{} stepConfig := StepConfig{Config: map[string]interface{}{}} stepParams := []StepParameters{stepParam(secretName, "vaultSecret", "$(vaultBasePath)/pipelineA")} - vaultData := map[string]string{secretName: "value1"} - vaultMock.On("GetKvSecret", "/pipelineA").Return(vaultData, nil) - err := addVaultCredentials(&stepConfig, vaultMock, stepParams) - assert.NoError(t, err) - assert.Equal(t, "value1", stepConfig.Config[secretName]) + addVaultCredentials(&stepConfig, vaultMock, stepParams) + assert.Equal(t, nil, stepConfig.Config[secretName]) + vaultMock.AssertNotCalled(t, "GetKvSecret", mock.AnythingOfType("string")) }) - } func stepParam(name string, refType string, refPaths ...string) StepParameters { diff --git a/pkg/vault/vault.go b/pkg/vault/client.go similarity index 94% rename from pkg/vault/vault.go rename to pkg/vault/client.go index 9617018a1..6df09e48d 100644 --- a/pkg/vault/vault.go +++ b/pkg/vault/client.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" + "github.com/SAP/jenkins-library/pkg/log" "github.com/hashicorp/vault/api" ) @@ -52,6 +53,7 @@ func NewClientWithAppRole(config *api.Config, roleID, secretID, namespace string client.SetNamespace(namespace) } + log.Entry().Debug("Using approle login") result, err := client.Logical().Write("auth/approle/login", map[string]interface{}{ "role_id": roleID, "secret_id": secretID, @@ -62,10 +64,11 @@ func NewClientWithAppRole(config *api.Config, roleID, secretID, namespace string } authInfo := result.Auth - if authInfo == nil { + if authInfo == nil || authInfo.ClientToken == "" { return Client{}, fmt.Errorf("Could not obtain token from approle with role_id %s", roleID) } + log.Entry().Debugf("Login to vault %s in namespace %s successfull", config.Address, namespace) return NewClient(config, authInfo.ClientToken, namespace) } diff --git a/pkg/vault/vault_test.go b/pkg/vault/client_test.go similarity index 100% rename from pkg/vault/vault_test.go rename to pkg/vault/client_test.go diff --git a/resources/metadata/checkmarx.yaml b/resources/metadata/checkmarx.yaml index 1b02a64c6..f69af44af 100644 --- a/resources/metadata/checkmarx.yaml +++ b/resources/metadata/checkmarx.yaml @@ -85,6 +85,11 @@ spec: - name: checkmarxCredentialsId type: secret param: password + - type: vaultSecret + paths: + - $(vaultPath)/checkmarx + - $(vaultBasePath)/$(vaultPipelineName)/checkmarx + - $(vaultBasePath)/GROUP-SECRETS/checkmarx - name: preset type: string description: The preset to use for scanning, if not set explicitly the step will attempt to look up the project's setting based on the availability of `checkmarxCredentialsId` @@ -163,6 +168,11 @@ spec: - name: checkmarxCredentialsId type: secret param: username + - type: vaultSecret + paths: + - $(vaultPath)/checkmarx + - $(vaultBasePath)/$(vaultPipelineName)/checkmarx + - $(vaultBasePath)/GROUP-SECRETS/checkmarx - name: verifyOnly type: bool description: Whether the step shall only apply verification checks or whether it does a full scan and check cycle diff --git a/resources/metadata/cloudFoundryCreateService.yaml b/resources/metadata/cloudFoundryCreateService.yaml index 246d19ed3..263eab011 100644 --- a/resources/metadata/cloudFoundryCreateService.yaml +++ b/resources/metadata/cloudFoundryCreateService.yaml @@ -46,6 +46,11 @@ spec: - name: cfCredentialsId type: secret param: username + - type: vaultSecret + paths: + - $(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace) - name: password type: string description: Password for Cloud Foundry User @@ -59,6 +64,11 @@ spec: - name: cfCredentialsId type: secret param: password + - type: vaultSecret + paths: + - $(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace) - name: cfOrg type: string description: Cloud Foundry org diff --git a/resources/metadata/cloudFoundryCreateServiceKey.yaml b/resources/metadata/cloudFoundryCreateServiceKey.yaml index 5f405ac42..1a14399bf 100644 --- a/resources/metadata/cloudFoundryCreateServiceKey.yaml +++ b/resources/metadata/cloudFoundryCreateServiceKey.yaml @@ -34,6 +34,11 @@ spec: - name: cfCredentialsId type: secret param: username + - type: vaultSecret + paths: + - $(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace) - name: password type: string description: User Password for CF User @@ -47,6 +52,11 @@ spec: - name: cfCredentialsId type: secret param: password + - type: vaultSecret + paths: + - $(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace) - name: cfOrg type: string description: CF org diff --git a/resources/metadata/cloudFoundryDeleteService.yaml b/resources/metadata/cloudFoundryDeleteService.yaml index c0b8086d3..fe261b34b 100644 --- a/resources/metadata/cloudFoundryDeleteService.yaml +++ b/resources/metadata/cloudFoundryDeleteService.yaml @@ -34,6 +34,11 @@ spec: - name: cfCredentialsId type: secret param: username + - type: vaultSecret + paths: + - $(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace) - name: password type: string description: User Password for CF User @@ -47,6 +52,11 @@ spec: - name: cfCredentialsId type: secret param: password + - type: vaultSecret + paths: + - $(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace) - name: cfOrg type: string description: CF org diff --git a/resources/metadata/cloudFoundryDeploy.yaml b/resources/metadata/cloudFoundryDeploy.yaml index 8372ca3cb..e3dadf4f3 100644 --- a/resources/metadata/cloudFoundryDeploy.yaml +++ b/resources/metadata/cloudFoundryDeploy.yaml @@ -297,6 +297,11 @@ spec: - name: cfCredentialsId type: secret param: password + - type: vaultSecret + paths: + - $(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace) - name: smokeTestScript type: string description: @@ -347,6 +352,11 @@ spec: - name: cfCredentialsId type: secret param: username + - type: vaultSecret + paths: + - $(vaultPath)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/$(vaultPipelineName)/cloudfoundry-$(cfOrg)-$(cfSpace) + - $(vaultBasePath)/GROUP-SECRETS/cloudfoundry-$(cfOrg)-$(cfSpace) containers: - name: cfDeploy image: ppiper/cf-cli diff --git a/resources/metadata/detect.yaml b/resources/metadata/detect.yaml index a34881f0a..60884350c 100644 --- a/resources/metadata/detect.yaml +++ b/resources/metadata/detect.yaml @@ -34,6 +34,11 @@ spec: resourceRef: - name: detectTokenCredentialsId type: secret + - type: vaultSecret + paths: + - $(vaultPath)/detect + - $(vaultBasePath)/$(vaultPipelineName)/detect + - $(vaultBasePath)/GROUP-SECRETS/detect scope: - PARAMETERS - STAGES diff --git a/resources/metadata/fortify.yaml b/resources/metadata/fortify.yaml index 33ddf144b..7b81002dd 100644 --- a/resources/metadata/fortify.yaml +++ b/resources/metadata/fortify.yaml @@ -42,6 +42,11 @@ spec: resourceRef: - name: fortifyCredentialsId type: secret + - type: vaultSecret + paths: + - $(vaultPath)/fortify + - $(vaultBasePath)/$(vaultPipelineName)/fortify + - $(vaultBasePath)/GROUP-SECRETS/fortify - name: githubToken description: "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line" diff --git a/resources/metadata/sonar.yaml b/resources/metadata/sonar.yaml index 5fe073e4d..7b3cedb3a 100644 --- a/resources/metadata/sonar.yaml +++ b/resources/metadata/sonar.yaml @@ -41,6 +41,11 @@ spec: - PARAMETERS secret: true resourceRef: + - type: vaultSecret + paths: + - $(vaultPath)/sonar + - $(vaultBasePath)/$(vaultPipelineName)/sonar + - $(vaultBasePath)/GROUP-SECRETS/sonar - name: sonarTokenCredentialsId type: secret aliases: diff --git a/resources/metadata/versioning.yaml b/resources/metadata/versioning.yaml index 8a2a2015f..61343d7a1 100644 --- a/resources/metadata/versioning.yaml +++ b/resources/metadata/versioning.yaml @@ -186,6 +186,11 @@ spec: - name: gitHttpsCredentialsId type: secret param: password + - type: vaultSecret + paths: + - $(vaultPath)/versioning + - $(vaultBasePath)/$(vaultPipelineName)/versioning + - $(vaultBasePath)/GROUP-SECRETS/versioning - name: projectSettingsFile aliases: - name: maven/projectSettingsFile @@ -230,6 +235,11 @@ spec: - name: gitHttpsCredentialsId type: secret param: username + - type: vaultSecret + paths: + - $(vaultPath)/versioning + - $(vaultBasePath)/$(vaultPipelineName)/versioning + - $(vaultBasePath)/GROUP-SECRETS/versioning - name: versioningTemplate type: string description: "DEPRECATED: Defines the template for the automatic version which will be created" diff --git a/vars/cloudFoundryDeploy.groovy b/vars/cloudFoundryDeploy.groovy index bf209531e..923fbdfdd 100644 --- a/vars/cloudFoundryDeploy.groovy +++ b/vars/cloudFoundryDeploy.groovy @@ -224,7 +224,7 @@ void call(Map parameters = [:]) { .dependingOn('deployTool').mixin('dockerWorkspace') .withMandatoryProperty('cloudFoundry/org') .withMandatoryProperty('cloudFoundry/space') - .withMandatoryProperty('cloudFoundry/credentialsId') + .withMandatoryProperty('cloudFoundry/credentialsId', null, {c -> return !c.containsKey('vaultAppRoleTokenCredentialsId') || !c.containsKey('vaultAppRoleSecretTokenCredentialsId')}) .use() if (config.useGoStep == true) { diff --git a/vars/piperExecuteBin.groovy b/vars/piperExecuteBin.groovy index 18ac5a873..b41200c76 100644 --- a/vars/piperExecuteBin.groovy +++ b/vars/piperExecuteBin.groovy @@ -156,6 +156,10 @@ void dockerWrapper(script, stepName, config, body) { // reused in sonarExecuteScan void credentialWrapper(config, List credentialInfo, body) { + if (config.containsKey('vaultAppRoleTokenCredentialsId') && config.containsKey('vaultAppRoleSecretTokenCredentialsId')) { + credentialInfo = [[type: 'token', id: 'vaultAppRoleTokenCredentialsId', env: ['PIPER_vaultAppRoleID']], + [type: 'token', id: 'vaultAppRoleSecretTokenCredentialsId', env: ['PIPER_vaultAppRoleSecretID']]] + } if (credentialInfo.size() > 0) { def creds = [] def sshCreds = []