(Vault) Improvements (#2439)

* vault improvements

* Update cloudFoundryDeploy.yaml

remove double PARAMETERS

* go generate

* fix type & resturcutre paragraph to a list

* remove non-existent secrets

* build trigger

Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>

cmd/piper.go

@@ -34,6 +34,10 @@ type GeneralConfigOptions struct {
 	LogFormat            string
 	VaultRoleID          string
 	VaultRoleSecretID    string
+	VaultToken           string
+	VaultServerURL       string
+	VaultNamespace       string
+	VaultPath            string
 	HookConfig           HookConfiguration
 }
 
@@ -149,6 +153,9 @@ func addRootFlags(rootCmd *cobra.Command) {
 	rootCmd.PersistentFlags().BoolVar(&GeneralConfig.NoTelemetry, "noTelemetry", false, "Disables telemetry reporting")
 	rootCmd.PersistentFlags().BoolVarP(&GeneralConfig.Verbose, "verbose", "v", false, "verbose output")
 	rootCmd.PersistentFlags().StringVar(&GeneralConfig.LogFormat, "logFormat", "default", "Log format to use. Options: default, timestamp, plain, full.")
+	rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultServerURL, "vaultServerUrl", "", "The vault server which should be used to fetch credentials")
+	rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultNamespace, "vaultNamespace", "", "The vault namespace which should be used to fetch credentials")
+	rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultPath, "vaultPath", "", "The path which should be used to fetch credentials")
 
 }
 
@@ -226,7 +233,10 @@ func PrepareConfig(cmd *cobra.Command, metadata *config.StepData, stepName strin
 	if GeneralConfig.VaultRoleSecretID == "" {
 		GeneralConfig.VaultRoleSecretID = os.Getenv("PIPER_vaultAppRoleSecretID")
 	}
-	myConfig.SetVaultCredentials(GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID)
+	if GeneralConfig.VaultToken == "" {
+		GeneralConfig.VaultToken = os.Getenv("PIPER_vaultToken")
+	}
+	myConfig.SetVaultCredentials(GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID, GeneralConfig.VaultToken)
 
 	if len(GeneralConfig.StepConfigJSON) != 0 {
 		// ignore config & defaults in favor of passed stepConfigJSON

documentation/docs/infrastructure/vault.md

@@ -1,25 +1,50 @@
 # Vault for Pipeline Secrets
 
-Project "Piper" also supports fetching your pipeline secrets directly from [Vault](https://www.hashicorp.com/products/vault).
+Project "Piper" supports fetching your pipeline secrets directly from [Vault](https://www.hashicorp.com/products/vault).
-Currently Vault's key value engine is supported in version 1 and 2, although we recommend version 2 since it supports versioning of secrets
+Currently, Vault's key value engine is supported in version 1 and 2, although we recommend version 2 since it supports
+the versioning of secrets
 
 Parameters that support being fetched from Vault are marked with the Vault Label in the Step Documentation.
 
 ![Vault Label](../images/parameter-with-vault-support.png)
 
-## Vault Setup
+## Authenticating Piper to Vault
 
-The first step to store your pipeline secrets in vault, is to enable a the [Key-Value Engine](https://www.vaultproject.io/docs/secrets/kv/kv-v2). And then create a policy which grants read access to the key value engine.
+Piper currently supports Vault's `AppRole` and `Token` authentication. However, `AppRole` authentication is recommended
-For Piper to authenticate against Vault, [AppRole](https://www.vaultproject.io/docs/auth/approle) authentication must be enabled in your Vault instance.
+since Piper is able to regularly rotate the SecretID, which is not possible with a Token.
-You have to [create an AppRole Role](https://www.vaultproject.io/api-docs/auth/approle#create-update-approle) for Piper and assign it the necessary policies.
 
-## Store Your Vault Credentials In Jenkins
+### AppRole Authentication
 
-Take the role ID from your Vault AppRole and create a Jenkins `Secret Text` credential. Do the same for the Vault AppRole secret ID.
+To authenticate against Vault, using [AppRole](https://www.vaultproject.io/docs/auth/approle) authentication you need to
+do the following things
+
+- Enable AppRole authentication in your vault instance.
+- After that you have
+  to [create an AppRole Role](https://www.vaultproject.io/api-docs/auth/approle#create-update-approle) for Piper
+- Assign the necessary policies to your newly created AppRole.
+- Take the **AppRole ID** and create a Jenkins `Secret Text` credential.
+- Take the **AppRole Secret ID** and create a Jenkins `Secret Text` credential.
 
 ![Create two jenkins secret text credentials](../images/jenkins-vault-credential.png)
 
-## Pipline Configuration
+### Token Authentication
+
+First step to use Token authentication is
+to [Create a vault Token](https://www.vaultproject.io/api/auth/token#create-token)
+In order to use a Vault Token for authentication you need to store the vault token inside your Jenkins instance as shown
+below.
+
+![Create a Jenkins secret text credential](../images/jenkins-vault-token-credential.png)
+
+## Setup a Secret Store in Vault
+
+The first step to store your pipeline secrets in Vault, is to enable a the
+[Key-Value Engine](https://www.vaultproject.io/docs/secrets/kv/kv-v2). Then create a policy which grants read access to
+the key value engine.
+
+![Enable a new secret engine in vault](../images/vault-secret-engine-enable.png)
+
+## Pipeline Configuration
 
 For pipelines to actually use the secrets stored in Vault you need to adjust your `config.yml`
 
@@ -33,3 +58,48 @@ general:
   vaultNamespace: '<YOUR_NAMESPACE_NAME>' # if you are not using vault's namespace feature you can remove this line
   ...
 ```
+
+Or if you chose to use Vault's token authentication then your  `config.yml` should look something like this.
+
+```yaml
+general:
+...
+vaultTokenCredentialsId: '<JENKINS_CREDENTIAL_ID_FOR_YOUR_VAULT_TOKEN>'
+vaultPath: 'kv/my-pipeline' # the path under which your jenkins secrets are stored
+vaultServerUrl: '<YOUR_VAULT_SERVER_URL>'
+vaultNamespace: '<YOUR_NAMESPACE_NAME>' # if you are not using vault's namespace feature you can remove this line
+...
+```
+
+## Configuring the Secret Lookup
+
+When Piper is configured to lookup secrets in Vault, there are some aspects that need to be considered.
+
+### Overwriting of Parameters
+
+Whenever a parameter is provided via `config.yml` or passed to the CLI it gets overwritten when a secret is found in
+Vault. To disable overriding parameters put a `vaultDisableOverwrite: false` on `Step` `Stage` or `General` Section in
+your config.
+
+```yaml
+general:
+  ...
+  vaultDisableOverwrite: true
+  ...
+steps:
+  executeBuild:
+    vaultDisableOverwrite: false
+    ...
+```
+
+### Skipping Vault Secret Lookup
+
+It is also possible to skip Vault for `Steps`, `Stages` or in `General` by using the `skipVault` config parameter as
+shown below.
+
+```yaml
+...
+steps:
+  executeBuild:
+    skipVault: true   # Skip Vault Secret Lookup for this step
+```

pkg/config/config.go

@@ -235,13 +235,16 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri
 	}
 
 	stepConfig.mixinVaultConfig(c.General, c.Steps[stepName], c.Stages[stageName])
-	// fetch secrets from vault
+	// check whether vault should be skipped
-	vaultClient, err := getVaultClientFromConfig(stepConfig, c.vaultCredentials)
+	if skip, ok := stepConfig.Config["skipVault"].(bool); !ok || !skip {
-	if err != nil {
+		// fetch secrets from vault
-		return StepConfig{}, err
+		vaultClient, err := getVaultClientFromConfig(stepConfig, c.vaultCredentials)
-	}
+		if err != nil {
-	if vaultClient != nil {
+			return StepConfig{}, err
-		resolveAllVaultReferences(&stepConfig, vaultClient, parameters)
+		}
+		if vaultClient != nil {
+			resolveAllVaultReferences(&stepConfig, vaultClient, parameters)
+		}
 	}
 
 	// finally do the condition evaluation post processing
@@ -262,11 +265,14 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri
 	return stepConfig, nil
 }
 
-// SetVaultCredentials sets the appRoleID and the appRoleSecretID to load additional configuration from vault
+// SetVaultCredentials sets the appRoleID and the appRoleSecretID or the vaultTokento load additional
-func (c *Config) SetVaultCredentials(appRoleID, appRoleSecretID string) {
+//configuration from vault
+// Either appRoleID and appRoleSecretID or vaultToken must be specified.
+func (c *Config) SetVaultCredentials(appRoleID, appRoleSecretID string, vaultToken string) {
 	c.vaultCredentials = VaultCredentials{
 		AppRoleID:       appRoleID,
 		AppRoleSecretID: appRoleSecretID,
+		VaultToken:      vaultToken,
 	}
 }
 

pkg/config/interpolation/interpolation.go

@@ -46,6 +46,7 @@ func resolveString(str string, lookupMap map[string]interface{}, n int) (string,
 			str = strings.ReplaceAll(str, fmt.Sprintf("$(%s)", property), propVal.(string))
 		} else {
 			// value not found
+			log.Entry().Debugf("Can't interploate '%s'. Missing property '%s'", str, property)
 			return "", false
 		}
 	}

pkg/config/stepmeta.go

@@ -233,7 +233,8 @@ func (m *StepData) GetContextParameterFilters() StepFilters {
 	}
 
 	if m.HasReference("vaultSecret") {
-		contextFilters = append(contextFilters, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}...)
+		contextFilters = append(contextFilters, []string{"vaultAppRoleTokenCredentialsId",
+			"vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}...)
 	}
 
 	if len(contextFilters) > 0 {

pkg/config/stepmeta_test.go

@@ -300,12 +300,12 @@ func TestGetContextParameterFilters(t *testing.T) {
 
 	t.Run("Vault", func(t *testing.T) {
 		filters := metadata4.GetContextParameterFilters()
-		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.All, "incorrect filter All")
+		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.All, "incorrect filter All")
-		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.General, "incorrect filter General")
+		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.General, "incorrect filter General")
-		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Steps, "incorrect filter Steps")
+		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.Steps, "incorrect filter Steps")
-		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Stages, "incorrect filter Stages")
+		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.Stages, "incorrect filter Stages")
-		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Parameters, "incorrect filter Parameters")
+		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.Parameters, "incorrect filter Parameters")
-		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Env, "incorrect filter Env")
+		assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.Env, "incorrect filter Env")
 	})
 }
 

pkg/config/vault.go

@@ -19,6 +19,8 @@ var (
 		"vaultBasePath",
 		"vaultPipelineName",
 		"vaultPath",
+		"skipVault",
+		"vaultDisableOverwrite",
 	}
 
 	// VaultSecretFileDirectory holds the directory for the current step run to temporarily store secret files fetched from vault
@@ -29,6 +31,7 @@ var (
 type VaultCredentials struct {
 	AppRoleID       string
 	AppRoleSecretID string
+	VaultToken      string
 }
 
 // vaultClient interface for mocking
@@ -45,7 +48,8 @@ func (s *StepConfig) mixinVaultConfig(configs ...map[string]interface{}) {
 func getVaultClientFromConfig(config StepConfig, creds VaultCredentials) (vaultClient, error) {
 	address, addressOk := config.Config["vaultServerUrl"].(string)
 	// if vault isn't used it's not an error
-	if !addressOk || creds.AppRoleID == "" || creds.AppRoleSecretID == "" {
+
+	if !addressOk || creds.VaultToken == "" && (creds.AppRoleID == "" || creds.AppRoleSecretID == "") {
 		log.Entry().Debug("Skipping fetching secrets from vault since it is not configured")
 		return nil, nil
 	}
@@ -56,21 +60,26 @@ func getVaultClientFromConfig(config StepConfig, creds VaultCredentials) (vaultC
 		log.Entry().Debugf("Using vault namespace %s", namespace)
 	}
 
-	client, err := vault.NewClientWithAppRole(&vault.Config{Config: &api.Config{Address: address}, Namespace: namespace}, creds.AppRoleID, creds.AppRoleSecretID)
+	var client vaultClient
+	var err error
+	clientConfig := &vault.Config{Config: &api.Config{Address: address}, Namespace: namespace}
+	if creds.VaultToken != "" {
+		log.Entry().Debugf("Using Vault Token Authentication")
+		client, err = vault.NewClient(clientConfig, creds.VaultToken)
+	} else {
+		log.Entry().Debugf("Using Vaults AppRole Authentication")
+		client, err = vault.NewClientWithAppRole(clientConfig, creds.AppRoleID, creds.AppRoleSecretID)
+	}
 	if err != nil {
 		return nil, err
 	}
 
 	log.Entry().Infof("Fetching secrets from vault at %s", address)
-	return &client, nil
+	return client, nil
 }
 
 func resolveAllVaultReferences(config *StepConfig, client vaultClient, params []StepParameters) {
 	for _, param := range params {
-		// we don't overwrite secrets that have already been set in any way
-		if _, ok := config.Config[param.Name].(string); ok {
-			continue
-		}
 		if ref := param.GetReference("vaultSecret"); ref != nil {
 			resolveVaultReference(ref, config, client, param)
 		}
@@ -81,6 +90,12 @@ func resolveAllVaultReferences(config *StepConfig, client vaultClient, params []
 }
 
 func resolveVaultReference(ref *ResourceReference, config *StepConfig, client vaultClient, param StepParameters) {
+	vaultDisableOverwrite, _ := config.Config["vaultDisableOverwrite"].(bool)
+	if _, ok := config.Config[param.Name].(string); vaultDisableOverwrite && ok {
+		log.Entry().Debugf("Not fetching '%s' from vault since it has already been set", param.Name)
+		return
+	}
+
 	var secretValue *string
 	for _, vaultPath := range ref.Paths {
 		// it should be possible to configure the root path were the secret is stored

pkg/config/vault_test.go

@@ -28,6 +28,21 @@ func TestVaultConfigLoad(t *testing.T) {
 	})
 
 	t.Run("Secrets are not overwritten", func(t *testing.T) {
+		vaultMock := &mocks.VaultMock{}
+		stepConfig := StepConfig{Config: map[string]interface{}{
+			"vaultBasePath":         "team1",
+			secretName:              "preset value",
+			"vaultDisableOverwrite": true,
+		}}
+		stepParams := []StepParameters{stepParam(secretName, "vaultSecret", "$(vaultBasePath)/pipelineA")}
+		vaultData := map[string]string{secretName: "value1"}
+		vaultMock.On("GetKvSecret", "team1/pipelineA").Return(vaultData, nil)
+		resolveAllVaultReferences(&stepConfig, vaultMock, stepParams)
+
+		assert.Equal(t, "preset value", stepConfig.Config[secretName])
+	})
+
+	t.Run("Secrets can be overwritten", func(t *testing.T) {
 		vaultMock := &mocks.VaultMock{}
 		stepConfig := StepConfig{Config: map[string]interface{}{
 			"vaultBasePath": "team1",
@@ -38,7 +53,7 @@ func TestVaultConfigLoad(t *testing.T) {
 		vaultMock.On("GetKvSecret", "team1/pipelineA").Return(vaultData, nil)
 		resolveAllVaultReferences(&stepConfig, vaultMock, stepParams)
 
-		assert.Equal(t, "preset value", stepConfig.Config[secretName])
+		assert.Equal(t, "value1", stepConfig.Config[secretName])
 	})
 
 	t.Run("Error is passed through", func(t *testing.T) {

pkg/vault/client.go

@@ -46,6 +46,7 @@ func NewClient(config *Config, token string) (Client, error) {
 	}
 
 	client.SetToken(token)
+	log.Entry().Debugf("Login to vault %s in namespace %s successfull", config.Address, config.Namespace)
 	return Client{client.Logical(), config}, nil
 }
 
@@ -83,7 +84,6 @@ func NewClientWithAppRole(config *Config, roleID, secretID string) (Client, erro
 		return Client{}, fmt.Errorf("Could not obtain token from approle with role_id %s", roleID)
 	}
 
-	log.Entry().Debugf("Login to vault %s in namespace %s successfull", config.Address, config.Namespace)
 	return NewClient(config, authInfo.ClientToken)
 }
 

vars/piperExecuteBin.groovy

@@ -28,6 +28,7 @@ void call(Map parameters = [:], String stepName, String metadataFile, List crede
         prepareExecution(script, utils, parameters)
         prepareMetadataResource(script, metadataFile)
         Map stepParameters = prepareStepParameters(parameters)
+        echo "Step params $stepParameters"
 
         withEnv([
             "PIPER_parametersJSON=${groovy.json.JsonOutput.toJson(stepParameters)}",
@@ -155,10 +156,8 @@ void dockerWrapper(script, stepName, config, body) {
 
 // reused in sonarExecuteScan
 void credentialWrapper(config, List credentialInfo, body) {
-    if (config.containsKey('vaultAppRoleTokenCredentialsId') && config.containsKey('vaultAppRoleSecretTokenCredentialsId')) {
+    credentialInfo = handleVaultCredentials(config, credentialInfo)
-        credentialInfo = [[type: 'token', id: 'vaultAppRoleTokenCredentialsId', env: ['PIPER_vaultAppRoleID']],
+
-                            [type: 'token', id: 'vaultAppRoleSecretTokenCredentialsId', env: ['PIPER_vaultAppRoleSecretID']]]
-    }
     if (credentialInfo.size() > 0) {
         def creds = []
         def sshCreds = []
@@ -170,7 +169,7 @@ void credentialWrapper(config, List credentialInfo, body) {
                 credentialsId = config[cred.id]
             }
             if (credentialsId) {
-                switch(cred.type) {
+                switch (cred.type) {
                     case "file":
                         creds.add(file(credentialsId: credentialsId, variable: cred.env[0]))
                         break
@@ -184,11 +183,17 @@ void credentialWrapper(config, List credentialInfo, body) {
                         sshCreds.add(credentialsId)
                         break
                     default:
-                        error ("invalid credential type: ${cred.type}")
+                        error("invalid credential type: ${cred.type}")
                 }
             }
         }
 
+        // remove credentialIds that were probably defaulted and which are not present in jenkins
+        if (containsVaultConfig(config)) {
+            creds = removeMissingCredentials(creds, config)
+            sshCreds = removeMissingCredentials(sshCreds, config)
+        }
+
         if (sshCreds.size() > 0) {
             sshagent (sshCreds) {
                 withCredentials(creds) {
@@ -205,6 +210,41 @@ void credentialWrapper(config, List credentialInfo, body) {
     }
 }
 
+List removeMissingCredentials(List creds, Map config) {
+    return creds.findAll { credentialExists(it, config) }
+}
+
+boolean credentialExists(cred, Map config) {
+    try {
+        withCredentials([cred]) {
+            return true
+        }
+    } catch (e) {
+        return false
+    }
+}
+
+boolean containsVaultConfig(Map config) {
+    def approleIsUsed = config.containsKey('vaultAppRoleTokenCredentialsId') && config.containsKey('vaultAppRoleSecretTokenCredentialsId')
+    def tokenIsUsed = config.containsKey('vaultTokenCredentialsId')
+
+    return approleIsUsed || tokenIsUsed
+}
+
+// Injects vaultCredentials if steps supports resolving parameters from vault
+List handleVaultCredentials(config, List credentialInfo) {
+    if (config.containsKey('vaultAppRoleTokenCredentialsId') && config.containsKey('vaultAppRoleSecretTokenCredentialsId')) {
+        credentialInfo += [[type: 'token', id: 'vaultAppRoleTokenCredentialsId', env: ['PIPER_vaultAppRoleID']],
+                            [type: 'token', id: 'vaultAppRoleSecretTokenCredentialsId', env: ['PIPER_vaultAppRoleSecretID']]]
+    }
+
+    if (config.containsKey('vaultTokenCredentialsId')) {
+        credentialInfo += [[type: 'token', id: 'vaultTokenCredentialsId', env: ['PIPER_vaultToken']]]
+    }
+
+    return credentialInfo
+}
+
 // reused in sonarExecuteScan
 void handleErrorDetails(String stepName, Closure body) {
     try {
@@ -218,7 +258,7 @@ void handleErrorDetails(String stepName, Closure body) {
                 errorCategory = " (category: ${errorDetails.category})"
                 DebugReport.instance.failedBuild.category = errorDetails.category
             }
-            error "[${stepName}] Step execution failed${errorCategory}. Error: ${errorDetails.error?:errorDetails.message}"
+            error "[${stepName}] Step execution failed${errorCategory}. Error: ${errorDetails.error ?: errorDetails.message}"
         }
         error "[${stepName}] Step execution failed. Error: ${ex}, please see log file for more details."
     }