From 3a0e7e3e86787c724335439810af7b7b7f49fb76 Mon Sep 17 00:00:00 2001 From: Anil Keshav Date: Tue, 21 Dec 2021 11:00:13 +0100 Subject: [PATCH] fetch general purpose credentials from vault (#3380) Co-authored-by: anilkeshav27 --- documentation/docs/infrastructure/vault.md | 32 ++++++- pkg/config/config.go | 1 + pkg/config/vault.go | 101 +++++++++++++++++++++ pkg/config/vault_test.go | 43 ++++++++- 4 files changed, 173 insertions(+), 4 deletions(-) diff --git a/documentation/docs/infrastructure/vault.md b/documentation/docs/infrastructure/vault.md index 0ab737559..b55b7ff8c 100644 --- a/documentation/docs/infrastructure/vault.md +++ b/documentation/docs/infrastructure/vault.md @@ -95,7 +95,37 @@ steps: skipVault: true # Skip Vault Secret Lookup for this step ``` -## Using vault for test credentials +## Using vault for general purpose and test credentials + +Vault can be used with piper to fetch any credentials, e.g. when they need to be appended to custom piper extensions or when they need to be appended to test command. The configuration for vault general purpose credentials can be added to **any** piper golang-based step. The configuration has to be done as follows: + +```yaml +general: + < your vault configuration > # see above +... +steps: + < piper go step >: + vaultCredentialPath: 'myStepCredentials' + vaultCredentialKeys: ['myAppId', 'myAppSecret'] +``` + +The `vaultCredentialPath` parameter is the endpoint of your credential path in vault. Depending on your _general_ config, the lookup for the credential IDs will be done in the following order respectively locations. The first path with found general purpose credentials will be used. + +1. `/` +2. `//` +3. `/GROUP-SECRETS/` + +The `vaultCredentialKeys`parameter is a list of credential IDs. The secret value of the credential will be exposed as an environment variable prefixed by "PIPER_VAULTCREDENTIAL_" and transformed to a valid variable name. For a credential ID named `myAppId` the forwarded environment variable to the step will be `PIPER_VAULTCREDENTIAL_MYAPPID` containing the secret. Hyphens will be replaced by underscores and other non-alphanumeric characters will be removed. + +!!! hint "Using a custom prefix for test credentials" + By default the prefix for test credentials is `PIPER_VAULTCREDENTIAL_`. + + It is possible to use a custom prefix by setting for example `vaultCredentialEnvPrefix: MY_CUSTOM_PREFIX` in your configuration. + With this above credential ID named `myAppId` will be populated into an environment variable with the name `MY_CUSTOM_PREFIX_MYAPPID`. + +Extended logging for vault secret fetching (e.g. found credentials and environment variable names) can be activated via `verbose: true` configuration. + +## Using vault for test credentials (Deprecated : use general purpose and test credentials as above) Vault can be used with piper to fetch any credentials, e.g. when they need to be appended to test command. The configuration for vault test credentials can be added to **any** piper golang-based step. The configuration has to be done as follows: diff --git a/pkg/config/config.go b/pkg/config/config.go index c2865e5b2..bdf673e44 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -265,6 +265,7 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri defer vaultClient.MustRevokeToken() resolveAllVaultReferences(&stepConfig, vaultClient, append(parameters, ReportingParameters.Parameters...)) resolveVaultTestCredentials(&stepConfig, vaultClient) + resolveVaultCredentials(&stepConfig, vaultClient) } } diff --git a/pkg/config/vault.go b/pkg/config/vault.go index 8fec8a11b..4273271a2 100644 --- a/pkg/config/vault.go +++ b/pkg/config/vault.go @@ -16,7 +16,9 @@ import ( const ( vaultRootPaths = "vaultRootPaths" vaultTestCredentialPath = "vaultTestCredentialPath" + vaultCredentialPath = "vaultCredentialPath" vaultTestCredentialKeys = "vaultTestCredentialKeys" + vaultCredentialKeys = "vaultCredentialKeys" vaultAppRoleID = "vaultAppRoleID" vaultAppRoleSecretID = "vaultAppRoleSecreId" vaultServerUrl = "vaultServerUrl" @@ -27,7 +29,9 @@ const ( skipVault = "skipVault" vaultDisableOverwrite = "vaultDisableOverwrite" vaultTestCredentialEnvPrefix = "vaultTestCredentialEnvPrefix" + vaultCredentialEnvPrefix = "vaultCredentialEnvPrefix" vaultTestCredentialEnvPrefixDefault = "PIPER_TESTCREDENTIAL_" + vaultCredentialEnvPrefixDefault = "PIPER_VAULTCREDENTIAL_" ) var ( @@ -45,6 +49,9 @@ var ( vaultTestCredentialPath, vaultTestCredentialKeys, vaultTestCredentialEnvPrefix, + vaultCredentialPath, + vaultCredentialKeys, + vaultCredentialEnvPrefix, } // VaultRootPaths are the lookup paths piper tries to use during the vault lookup. @@ -200,6 +207,43 @@ func resolveVaultTestCredentials(config *StepConfig, client vaultClient) { } } +func resolveVaultCredentials(config *StepConfig, client vaultClient) { + credPath, pathOk := config.Config[vaultCredentialPath].(string) + keys := getCredentialKeys(config) + if !(pathOk && keys != nil) || credPath == "" || len(keys) == 0 { + log.Entry().Debugf("Not fetching test credentials from vault since they are not (properly) configured") + return + } + + lookupPath := make([]string, 3) + lookupPath[0] = "$(vaultPath)/" + credPath + lookupPath[1] = "$(vaultBasePath)/$(vaultPipelineName)/" + credPath + lookupPath[2] = "$(vaultBasePath)/GROUP-SECRETS/" + credPath + + for _, path := range lookupPath { + vaultPath, ok := interpolation.ResolveString(path, config.Config) + if !ok { + continue + } + + secret, err := client.GetKvSecret(vaultPath) + if err != nil { + log.Entry().WithError(err).Debugf("Couldn't fetch secret at '%s'", vaultPath) + continue + } + if secret == nil { + continue + } + secretsResolved := false + secretsResolved = populateCredentialsAsEnvs(config, secret, keys) + if secretsResolved { + // prevent overwriting resolved secrets + // only allows vault test credentials on one / the same vault path + break + } + } +} + func populateTestCredentialsAsEnvs(config *StepConfig, secret map[string]string, keys []string) (matched bool) { vaultTestCredentialEnvPrefix, ok := config.Config["vaultTestCredentialEnvPrefix"].(string) @@ -220,6 +264,45 @@ func populateTestCredentialsAsEnvs(config *StepConfig, secret map[string]string, return } +func populateCredentialsAsEnvs(config *StepConfig, secret map[string]string, keys []string) (matched bool) { + + vaultCredentialEnvPrefix, ok := config.Config["vaultCredentialEnvPrefix"].(string) + isCredentialEnvPrefixDefault := false + + if !ok || len(vaultCredentialEnvPrefix) == 0 { + vaultCredentialEnvPrefix = vaultCredentialEnvPrefixDefault + isCredentialEnvPrefixDefault = true + } + for secretKey, secretValue := range secret { + for _, key := range keys { + if secretKey == key { + log.RegisterSecret(secretValue) + envVariable := vaultCredentialEnvPrefix + convertEnvVar(secretKey) + log.Entry().Debugf("Exposing general purpose credential '%v' as '%v'", key, envVariable) + os.Setenv(envVariable, secretValue) + matched = true + } + } + } + + // we always create a standard env variable with the default prefx so that + // we can always refer to it in steps if its to be hard-coded + if !isCredentialEnvPrefixDefault { + for secretKey, secretValue := range secret { + for _, key := range keys { + if secretKey == key { + log.RegisterSecret(secretValue) + envVariable := vaultCredentialEnvPrefixDefault + convertEnvVar(secretKey) + log.Entry().Debugf("Exposing general purpose credential '%v' as '%v'", key, envVariable) + os.Setenv(envVariable, secretValue) + matched = true + } + } + } + } + return +} + func getTestCredentialKeys(config *StepConfig) []string { keysRaw, ok := config.Config[vaultTestCredentialKeys].([]interface{}) if !ok { @@ -238,6 +321,24 @@ func getTestCredentialKeys(config *StepConfig) []string { return keys } +func getCredentialKeys(config *StepConfig) []string { + keysRaw, ok := config.Config[vaultCredentialKeys].([]interface{}) + if !ok { + log.Entry().Debugf("Not fetching general purpose credentials from vault since they are not (properly) configured") + return nil + } + keys := make([]string, 0, len(keysRaw)) + for _, keyRaw := range keysRaw { + key, ok := keyRaw.(string) + if !ok { + log.Entry().Warnf("%s is needs to be an array of strings", vaultCredentialKeys) + return nil + } + keys = append(keys, key) + } + return keys +} + // converts to a valid environment variable string func convertEnvVar(s string) string { r := strings.ToUpper(s) diff --git a/pkg/config/vault_test.go b/pkg/config/vault_test.go index 4f00ca2dc..6bfe1036e 100644 --- a/pkg/config/vault_test.go +++ b/pkg/config/vault_test.go @@ -2,13 +2,14 @@ package config import ( "fmt" - "github.com/stretchr/testify/mock" "io/ioutil" "os" "path" "strings" "testing" + "github.com/stretchr/testify/mock" + "github.com/SAP/jenkins-library/pkg/config/mocks" "github.com/stretchr/testify/assert" ) @@ -225,7 +226,7 @@ func addAlias(param *StepParameters, aliasName string) { func TestResolveVaultTestCredentials(t *testing.T) { t.Parallel() - t.Run("Default credential prefix", func(t *testing.T) { + t.Run("Default test credential prefix", func(t *testing.T) { t.Parallel() // init vaultMock := &mocks.VaultMock{} @@ -254,7 +255,43 @@ func TestResolveVaultTestCredentials(t *testing.T) { } }) - t.Run("Custom credential prefix", func(t *testing.T) { + t.Run("Custom general purpose credential prefix along with fixed standard prefix", func(t *testing.T) { + t.Parallel() + // init + vaultMock := &mocks.VaultMock{} + envPrefix := "CUSTOM_MYCRED_" + standardEnvPrefix := "PIPER_VAULTCREDENTIAL_" + stepConfig := StepConfig{Config: map[string]interface{}{ + "vaultPath": "team1", + "vaultCredentialPath": "appCredentials", + "vaultCredentialKeys": []interface{}{"appUser", "appUserPw"}, + "vaultCredentialEnvPrefix": envPrefix, + }} + + defer os.Unsetenv("CUSTOM_MYCRED_APPUSER") + defer os.Unsetenv("CUSTOM_MYCRED_APPUSERPW") + defer os.Unsetenv("PIPER_VAULTCREDENTIAL_APPUSER") + defer os.Unsetenv("PIPER_VAULTCREDENTIAL_APPUSERPW") + + // mock + vaultData := map[string]string{"appUser": "test-user", "appUserPw": "password1234"} + vaultMock.On("GetKvSecret", "team1/appCredentials").Return(vaultData, nil) + + // test + resolveVaultCredentials(&stepConfig, vaultMock) + + // assert + for k, v := range vaultData { + env := envPrefix + strings.ToUpper(k) + assert.NotEmpty(t, os.Getenv(env)) + assert.Equal(t, os.Getenv(env), v) + standardEnv := standardEnvPrefix + strings.ToUpper(k) + assert.NotEmpty(t, os.Getenv(standardEnv)) + assert.Equal(t, os.Getenv(standardEnv), v) + } + }) + + t.Run("Custom test credential prefix", func(t *testing.T) { t.Parallel() // init vaultMock := &mocks.VaultMock{}