fetch general purpose credentials from vault (#3380)

Co-authored-by: anilkeshav27 <you@example.com>

documentation/docs/infrastructure/vault.md

@@ -95,7 +95,37 @@ steps:
     skipVault: true   # Skip Vault Secret Lookup for this step
 ```
 
-## Using vault for test credentials
+## Using vault for general purpose and test credentials
+
+Vault can be used with piper to fetch any credentials, e.g. when they need to be appended to custom piper extensions or when they need to be appended to test command. The configuration for vault general purpose credentials can be added to **any** piper golang-based step. The configuration has to be done as follows:
+
+```yaml
+general:
+  < your vault configuration > # see above
+...
+steps:
+  < piper go step >:
+    vaultCredentialPath: 'myStepCredentials'
+    vaultCredentialKeys: ['myAppId', 'myAppSecret']
+```
+
+The `vaultCredentialPath` parameter is the endpoint of your credential path in vault. Depending on your _general_ config, the lookup for the credential IDs will be done in the following order respectively locations. The first path with found general purpose credentials will be used.
+
+1. `<vaultPath>/<vaultCredentialPath>`
+2. `<vaultBasePath>/<vaultPipelineName>/<vaultCredentialPath>`
+3. `<vaultBasePath>/GROUP-SECRETS/<vaultCredentialPath>`
+
+The `vaultCredentialKeys`parameter is a list of credential IDs. The secret value of the credential will be exposed as an environment variable prefixed by "PIPER_VAULTCREDENTIAL_" and transformed to a valid variable name. For a credential ID named `myAppId` the forwarded environment variable to the step will be `PIPER_VAULTCREDENTIAL_MYAPPID` containing the secret. Hyphens will be replaced by underscores and other non-alphanumeric characters will be removed.
+
+!!! hint "Using a custom prefix for test credentials"
+    By default the prefix for test credentials is `PIPER_VAULTCREDENTIAL_`.
+
+    It is possible to use a custom prefix by setting for example `vaultCredentialEnvPrefix: MY_CUSTOM_PREFIX` in your configuration.
+    With this above credential ID named `myAppId` will be populated into an environment variable with the name `MY_CUSTOM_PREFIX_MYAPPID`.
+
+Extended logging for vault secret fetching (e.g. found credentials and environment variable names) can be activated via `verbose: true` configuration.
+
+## Using vault for test credentials (Deprecated : use general purpose and test credentials as above)
 
 Vault can be used with piper to fetch any credentials, e.g. when they need to be appended to test command. The configuration for vault test credentials can be added to **any** piper golang-based step. The configuration has to be done as follows:
 

pkg/config/config.go

@@ -265,6 +265,7 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri
 			defer vaultClient.MustRevokeToken()
 			resolveAllVaultReferences(&stepConfig, vaultClient, append(parameters, ReportingParameters.Parameters...))
 			resolveVaultTestCredentials(&stepConfig, vaultClient)
+			resolveVaultCredentials(&stepConfig, vaultClient)
 		}
 	}
 

pkg/config/vault.go

@@ -16,7 +16,9 @@ import (
 const (
 	vaultRootPaths                      = "vaultRootPaths"
 	vaultTestCredentialPath             = "vaultTestCredentialPath"
+	vaultCredentialPath                 = "vaultCredentialPath"
 	vaultTestCredentialKeys             = "vaultTestCredentialKeys"
+	vaultCredentialKeys                 = "vaultCredentialKeys"
 	vaultAppRoleID                      = "vaultAppRoleID"
 	vaultAppRoleSecretID                = "vaultAppRoleSecreId"
 	vaultServerUrl                      = "vaultServerUrl"
@@ -27,7 +29,9 @@ const (
 	skipVault                           = "skipVault"
 	vaultDisableOverwrite               = "vaultDisableOverwrite"
 	vaultTestCredentialEnvPrefix        = "vaultTestCredentialEnvPrefix"
+	vaultCredentialEnvPrefix            = "vaultCredentialEnvPrefix"
 	vaultTestCredentialEnvPrefixDefault = "PIPER_TESTCREDENTIAL_"
+	vaultCredentialEnvPrefixDefault     = "PIPER_VAULTCREDENTIAL_"
 )
 
 var (
@@ -45,6 +49,9 @@ var (
 		vaultTestCredentialPath,
 		vaultTestCredentialKeys,
 		vaultTestCredentialEnvPrefix,
+		vaultCredentialPath,
+		vaultCredentialKeys,
+		vaultCredentialEnvPrefix,
 	}
 
 	// VaultRootPaths are the lookup paths piper tries to use during the vault lookup.
@@ -200,6 +207,43 @@ func resolveVaultTestCredentials(config *StepConfig, client vaultClient) {
 	}
 }
 
+func resolveVaultCredentials(config *StepConfig, client vaultClient) {
+	credPath, pathOk := config.Config[vaultCredentialPath].(string)
+	keys := getCredentialKeys(config)
+	if !(pathOk && keys != nil) || credPath == "" || len(keys) == 0 {
+		log.Entry().Debugf("Not fetching test credentials from vault since they are not (properly) configured")
+		return
+	}
+
+	lookupPath := make([]string, 3)
+	lookupPath[0] = "$(vaultPath)/" + credPath
+	lookupPath[1] = "$(vaultBasePath)/$(vaultPipelineName)/" + credPath
+	lookupPath[2] = "$(vaultBasePath)/GROUP-SECRETS/" + credPath
+
+	for _, path := range lookupPath {
+		vaultPath, ok := interpolation.ResolveString(path, config.Config)
+		if !ok {
+			continue
+		}
+
+		secret, err := client.GetKvSecret(vaultPath)
+		if err != nil {
+			log.Entry().WithError(err).Debugf("Couldn't fetch secret at '%s'", vaultPath)
+			continue
+		}
+		if secret == nil {
+			continue
+		}
+		secretsResolved := false
+		secretsResolved = populateCredentialsAsEnvs(config, secret, keys)
+		if secretsResolved {
+			// prevent overwriting resolved secrets
+			// only allows vault test credentials on one / the same vault path
+			break
+		}
+	}
+}
+
 func populateTestCredentialsAsEnvs(config *StepConfig, secret map[string]string, keys []string) (matched bool) {
 
 	vaultTestCredentialEnvPrefix, ok := config.Config["vaultTestCredentialEnvPrefix"].(string)
@@ -220,6 +264,45 @@ func populateTestCredentialsAsEnvs(config *StepConfig, secret map[string]string,
 	return
 }
 
+func populateCredentialsAsEnvs(config *StepConfig, secret map[string]string, keys []string) (matched bool) {
+
+	vaultCredentialEnvPrefix, ok := config.Config["vaultCredentialEnvPrefix"].(string)
+	isCredentialEnvPrefixDefault := false
+
+	if !ok || len(vaultCredentialEnvPrefix) == 0 {
+		vaultCredentialEnvPrefix = vaultCredentialEnvPrefixDefault
+		isCredentialEnvPrefixDefault = true
+	}
+	for secretKey, secretValue := range secret {
+		for _, key := range keys {
+			if secretKey == key {
+				log.RegisterSecret(secretValue)
+				envVariable := vaultCredentialEnvPrefix + convertEnvVar(secretKey)
+				log.Entry().Debugf("Exposing general purpose credential '%v' as '%v'", key, envVariable)
+				os.Setenv(envVariable, secretValue)
+				matched = true
+			}
+		}
+	}
+
+	// we always create a standard env variable with the default prefx so that
+	// we can always refer to it in steps if its to be hard-coded
+	if !isCredentialEnvPrefixDefault {
+		for secretKey, secretValue := range secret {
+			for _, key := range keys {
+				if secretKey == key {
+					log.RegisterSecret(secretValue)
+					envVariable := vaultCredentialEnvPrefixDefault + convertEnvVar(secretKey)
+					log.Entry().Debugf("Exposing general purpose credential '%v' as '%v'", key, envVariable)
+					os.Setenv(envVariable, secretValue)
+					matched = true
+				}
+			}
+		}
+	}
+	return
+}
+
 func getTestCredentialKeys(config *StepConfig) []string {
 	keysRaw, ok := config.Config[vaultTestCredentialKeys].([]interface{})
 	if !ok {
@@ -238,6 +321,24 @@ func getTestCredentialKeys(config *StepConfig) []string {
 	return keys
 }
 
+func getCredentialKeys(config *StepConfig) []string {
+	keysRaw, ok := config.Config[vaultCredentialKeys].([]interface{})
+	if !ok {
+		log.Entry().Debugf("Not fetching general purpose credentials from vault since they are not (properly) configured")
+		return nil
+	}
+	keys := make([]string, 0, len(keysRaw))
+	for _, keyRaw := range keysRaw {
+		key, ok := keyRaw.(string)
+		if !ok {
+			log.Entry().Warnf("%s is needs to be an array of strings", vaultCredentialKeys)
+			return nil
+		}
+		keys = append(keys, key)
+	}
+	return keys
+}
+
 // converts to a valid environment variable string
 func convertEnvVar(s string) string {
 	r := strings.ToUpper(s)

pkg/config/vault_test.go

@@ -2,13 +2,14 @@ package config
 
 import (
 	"fmt"
-	"github.com/stretchr/testify/mock"
 	"io/ioutil"
 	"os"
 	"path"
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/mock"
+
 	"github.com/SAP/jenkins-library/pkg/config/mocks"
 	"github.com/stretchr/testify/assert"
 )
@@ -225,7 +226,7 @@ func addAlias(param *StepParameters, aliasName string) {
 
 func TestResolveVaultTestCredentials(t *testing.T) {
 	t.Parallel()
-	t.Run("Default credential prefix", func(t *testing.T) {
+	t.Run("Default test credential prefix", func(t *testing.T) {
 		t.Parallel()
 		// init
 		vaultMock := &mocks.VaultMock{}
@@ -254,7 +255,43 @@ func TestResolveVaultTestCredentials(t *testing.T) {
 		}
 	})
 
-	t.Run("Custom credential prefix", func(t *testing.T) {
+	t.Run("Custom general purpose credential prefix along with fixed standard prefix", func(t *testing.T) {
+		t.Parallel()
+		// init
+		vaultMock := &mocks.VaultMock{}
+		envPrefix := "CUSTOM_MYCRED_"
+		standardEnvPrefix := "PIPER_VAULTCREDENTIAL_"
+		stepConfig := StepConfig{Config: map[string]interface{}{
+			"vaultPath":                "team1",
+			"vaultCredentialPath":      "appCredentials",
+			"vaultCredentialKeys":      []interface{}{"appUser", "appUserPw"},
+			"vaultCredentialEnvPrefix": envPrefix,
+		}}
+
+		defer os.Unsetenv("CUSTOM_MYCRED_APPUSER")
+		defer os.Unsetenv("CUSTOM_MYCRED_APPUSERPW")
+		defer os.Unsetenv("PIPER_VAULTCREDENTIAL_APPUSER")
+		defer os.Unsetenv("PIPER_VAULTCREDENTIAL_APPUSERPW")
+
+		// mock
+		vaultData := map[string]string{"appUser": "test-user", "appUserPw": "password1234"}
+		vaultMock.On("GetKvSecret", "team1/appCredentials").Return(vaultData, nil)
+
+		// test
+		resolveVaultCredentials(&stepConfig, vaultMock)
+
+		// assert
+		for k, v := range vaultData {
+			env := envPrefix + strings.ToUpper(k)
+			assert.NotEmpty(t, os.Getenv(env))
+			assert.Equal(t, os.Getenv(env), v)
+			standardEnv := standardEnvPrefix + strings.ToUpper(k)
+			assert.NotEmpty(t, os.Getenv(standardEnv))
+			assert.Equal(t, os.Getenv(standardEnv), v)
+		}
+	})
+
+	t.Run("Custom test credential prefix", func(t *testing.T) {
 		t.Parallel()
 		// init
 		vaultMock := &mocks.VaultMock{}