diff --git a/cmd/piper.go b/cmd/piper.go index 9ec0dd9d0..4725d64bc 100644 --- a/cmd/piper.go +++ b/cmd/piper.go @@ -34,6 +34,10 @@ type GeneralConfigOptions struct { LogFormat string VaultRoleID string VaultRoleSecretID string + VaultToken string + VaultServerURL string + VaultNamespace string + VaultPath string HookConfig HookConfiguration } @@ -149,6 +153,9 @@ func addRootFlags(rootCmd *cobra.Command) { rootCmd.PersistentFlags().BoolVar(&GeneralConfig.NoTelemetry, "noTelemetry", false, "Disables telemetry reporting") rootCmd.PersistentFlags().BoolVarP(&GeneralConfig.Verbose, "verbose", "v", false, "verbose output") rootCmd.PersistentFlags().StringVar(&GeneralConfig.LogFormat, "logFormat", "default", "Log format to use. Options: default, timestamp, plain, full.") + rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultServerURL, "vaultServerUrl", "", "The vault server which should be used to fetch credentials") + rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultNamespace, "vaultNamespace", "", "The vault namespace which should be used to fetch credentials") + rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultPath, "vaultPath", "", "The path which should be used to fetch credentials") } @@ -226,7 +233,10 @@ func PrepareConfig(cmd *cobra.Command, metadata *config.StepData, stepName strin if GeneralConfig.VaultRoleSecretID == "" { GeneralConfig.VaultRoleSecretID = os.Getenv("PIPER_vaultAppRoleSecretID") } - myConfig.SetVaultCredentials(GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID) + if GeneralConfig.VaultToken == "" { + GeneralConfig.VaultToken = os.Getenv("PIPER_vaultToken") + } + myConfig.SetVaultCredentials(GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID, GeneralConfig.VaultToken) if len(GeneralConfig.StepConfigJSON) != 0 { // ignore config & defaults in favor of passed stepConfigJSON diff --git a/documentation/docs/images/jenkins-vault-token-credential.png b/documentation/docs/images/jenkins-vault-token-credential.png new file mode 100644 index 000000000..2cd81914d Binary files /dev/null and b/documentation/docs/images/jenkins-vault-token-credential.png differ diff --git a/documentation/docs/images/vault-secret-engine-enable.png b/documentation/docs/images/vault-secret-engine-enable.png new file mode 100644 index 000000000..08083c0e1 Binary files /dev/null and b/documentation/docs/images/vault-secret-engine-enable.png differ diff --git a/documentation/docs/infrastructure/vault.md b/documentation/docs/infrastructure/vault.md index 08c3de535..2fad7adbb 100644 --- a/documentation/docs/infrastructure/vault.md +++ b/documentation/docs/infrastructure/vault.md @@ -1,25 +1,50 @@ # Vault for Pipeline Secrets -Project "Piper" also supports fetching your pipeline secrets directly from [Vault](https://www.hashicorp.com/products/vault). -Currently Vault's key value engine is supported in version 1 and 2, although we recommend version 2 since it supports versioning of secrets +Project "Piper" supports fetching your pipeline secrets directly from [Vault](https://www.hashicorp.com/products/vault). +Currently, Vault's key value engine is supported in version 1 and 2, although we recommend version 2 since it supports +the versioning of secrets Parameters that support being fetched from Vault are marked with the Vault Label in the Step Documentation. ![Vault Label](../images/parameter-with-vault-support.png) -## Vault Setup +## Authenticating Piper to Vault -The first step to store your pipeline secrets in vault, is to enable a the [Key-Value Engine](https://www.vaultproject.io/docs/secrets/kv/kv-v2). And then create a policy which grants read access to the key value engine. -For Piper to authenticate against Vault, [AppRole](https://www.vaultproject.io/docs/auth/approle) authentication must be enabled in your Vault instance. -You have to [create an AppRole Role](https://www.vaultproject.io/api-docs/auth/approle#create-update-approle) for Piper and assign it the necessary policies. +Piper currently supports Vault's `AppRole` and `Token` authentication. However, `AppRole` authentication is recommended +since Piper is able to regularly rotate the SecretID, which is not possible with a Token. -## Store Your Vault Credentials In Jenkins +### AppRole Authentication -Take the role ID from your Vault AppRole and create a Jenkins `Secret Text` credential. Do the same for the Vault AppRole secret ID. +To authenticate against Vault, using [AppRole](https://www.vaultproject.io/docs/auth/approle) authentication you need to +do the following things + +- Enable AppRole authentication in your vault instance. +- After that you have + to [create an AppRole Role](https://www.vaultproject.io/api-docs/auth/approle#create-update-approle) for Piper +- Assign the necessary policies to your newly created AppRole. +- Take the **AppRole ID** and create a Jenkins `Secret Text` credential. +- Take the **AppRole Secret ID** and create a Jenkins `Secret Text` credential. ![Create two jenkins secret text credentials](../images/jenkins-vault-credential.png) -## Pipline Configuration +### Token Authentication + +First step to use Token authentication is +to [Create a vault Token](https://www.vaultproject.io/api/auth/token#create-token) +In order to use a Vault Token for authentication you need to store the vault token inside your Jenkins instance as shown +below. + +![Create a Jenkins secret text credential](../images/jenkins-vault-token-credential.png) + +## Setup a Secret Store in Vault + +The first step to store your pipeline secrets in Vault, is to enable a the +[Key-Value Engine](https://www.vaultproject.io/docs/secrets/kv/kv-v2). Then create a policy which grants read access to +the key value engine. + +![Enable a new secret engine in vault](../images/vault-secret-engine-enable.png) + +## Pipeline Configuration For pipelines to actually use the secrets stored in Vault you need to adjust your `config.yml` @@ -33,3 +58,48 @@ general: vaultNamespace: '' # if you are not using vault's namespace feature you can remove this line ... ``` + +Or if you chose to use Vault's token authentication then your `config.yml` should look something like this. + +```yaml +general: +... +vaultTokenCredentialsId: '' +vaultPath: 'kv/my-pipeline' # the path under which your jenkins secrets are stored +vaultServerUrl: '' +vaultNamespace: '' # if you are not using vault's namespace feature you can remove this line +... +``` + +## Configuring the Secret Lookup + +When Piper is configured to lookup secrets in Vault, there are some aspects that need to be considered. + +### Overwriting of Parameters + +Whenever a parameter is provided via `config.yml` or passed to the CLI it gets overwritten when a secret is found in +Vault. To disable overriding parameters put a `vaultDisableOverwrite: false` on `Step` `Stage` or `General` Section in +your config. + +```yaml +general: + ... + vaultDisableOverwrite: true + ... +steps: + executeBuild: + vaultDisableOverwrite: false + ... +``` + +### Skipping Vault Secret Lookup + +It is also possible to skip Vault for `Steps`, `Stages` or in `General` by using the `skipVault` config parameter as +shown below. + +```yaml +... +steps: + executeBuild: + skipVault: true # Skip Vault Secret Lookup for this step +``` diff --git a/pkg/config/config.go b/pkg/config/config.go index 1fc747fc1..8ed754aef 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -235,13 +235,16 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri } stepConfig.mixinVaultConfig(c.General, c.Steps[stepName], c.Stages[stageName]) - // fetch secrets from vault - vaultClient, err := getVaultClientFromConfig(stepConfig, c.vaultCredentials) - if err != nil { - return StepConfig{}, err - } - if vaultClient != nil { - resolveAllVaultReferences(&stepConfig, vaultClient, parameters) + // check whether vault should be skipped + if skip, ok := stepConfig.Config["skipVault"].(bool); !ok || !skip { + // fetch secrets from vault + vaultClient, err := getVaultClientFromConfig(stepConfig, c.vaultCredentials) + if err != nil { + return StepConfig{}, err + } + if vaultClient != nil { + resolveAllVaultReferences(&stepConfig, vaultClient, parameters) + } } // finally do the condition evaluation post processing @@ -262,11 +265,14 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri return stepConfig, nil } -// SetVaultCredentials sets the appRoleID and the appRoleSecretID to load additional configuration from vault -func (c *Config) SetVaultCredentials(appRoleID, appRoleSecretID string) { +// SetVaultCredentials sets the appRoleID and the appRoleSecretID or the vaultTokento load additional +//configuration from vault +// Either appRoleID and appRoleSecretID or vaultToken must be specified. +func (c *Config) SetVaultCredentials(appRoleID, appRoleSecretID string, vaultToken string) { c.vaultCredentials = VaultCredentials{ AppRoleID: appRoleID, AppRoleSecretID: appRoleSecretID, + VaultToken: vaultToken, } } diff --git a/pkg/config/interpolation/interpolation.go b/pkg/config/interpolation/interpolation.go index 53922a9b6..d9551ba18 100644 --- a/pkg/config/interpolation/interpolation.go +++ b/pkg/config/interpolation/interpolation.go @@ -46,6 +46,7 @@ func resolveString(str string, lookupMap map[string]interface{}, n int) (string, str = strings.ReplaceAll(str, fmt.Sprintf("$(%s)", property), propVal.(string)) } else { // value not found + log.Entry().Debugf("Can't interploate '%s'. Missing property '%s'", str, property) return "", false } } diff --git a/pkg/config/stepmeta.go b/pkg/config/stepmeta.go index cf8e1ed1b..bd12b33a1 100644 --- a/pkg/config/stepmeta.go +++ b/pkg/config/stepmeta.go @@ -233,7 +233,8 @@ func (m *StepData) GetContextParameterFilters() StepFilters { } if m.HasReference("vaultSecret") { - contextFilters = append(contextFilters, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}...) + contextFilters = append(contextFilters, []string{"vaultAppRoleTokenCredentialsId", + "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}...) } if len(contextFilters) > 0 { diff --git a/pkg/config/stepmeta_test.go b/pkg/config/stepmeta_test.go index 7992aad65..4ce5a5ba3 100644 --- a/pkg/config/stepmeta_test.go +++ b/pkg/config/stepmeta_test.go @@ -300,12 +300,12 @@ func TestGetContextParameterFilters(t *testing.T) { t.Run("Vault", func(t *testing.T) { filters := metadata4.GetContextParameterFilters() - assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.All, "incorrect filter All") - assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.General, "incorrect filter General") - assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Steps, "incorrect filter Steps") - assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Stages, "incorrect filter Stages") - assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Parameters, "incorrect filter Parameters") - assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId"}, filters.Env, "incorrect filter Env") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.All, "incorrect filter All") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.General, "incorrect filter General") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.Steps, "incorrect filter Steps") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.Stages, "incorrect filter Stages") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.Parameters, "incorrect filter Parameters") + assert.Equal(t, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}, filters.Env, "incorrect filter Env") }) } diff --git a/pkg/config/vault.go b/pkg/config/vault.go index 8398bcf92..482e74201 100644 --- a/pkg/config/vault.go +++ b/pkg/config/vault.go @@ -19,6 +19,8 @@ var ( "vaultBasePath", "vaultPipelineName", "vaultPath", + "skipVault", + "vaultDisableOverwrite", } // VaultSecretFileDirectory holds the directory for the current step run to temporarily store secret files fetched from vault @@ -29,6 +31,7 @@ var ( type VaultCredentials struct { AppRoleID string AppRoleSecretID string + VaultToken string } // vaultClient interface for mocking @@ -45,7 +48,8 @@ func (s *StepConfig) mixinVaultConfig(configs ...map[string]interface{}) { func getVaultClientFromConfig(config StepConfig, creds VaultCredentials) (vaultClient, error) { address, addressOk := config.Config["vaultServerUrl"].(string) // if vault isn't used it's not an error - if !addressOk || creds.AppRoleID == "" || creds.AppRoleSecretID == "" { + + if !addressOk || creds.VaultToken == "" && (creds.AppRoleID == "" || creds.AppRoleSecretID == "") { log.Entry().Debug("Skipping fetching secrets from vault since it is not configured") return nil, nil } @@ -56,21 +60,26 @@ func getVaultClientFromConfig(config StepConfig, creds VaultCredentials) (vaultC log.Entry().Debugf("Using vault namespace %s", namespace) } - client, err := vault.NewClientWithAppRole(&vault.Config{Config: &api.Config{Address: address}, Namespace: namespace}, creds.AppRoleID, creds.AppRoleSecretID) + var client vaultClient + var err error + clientConfig := &vault.Config{Config: &api.Config{Address: address}, Namespace: namespace} + if creds.VaultToken != "" { + log.Entry().Debugf("Using Vault Token Authentication") + client, err = vault.NewClient(clientConfig, creds.VaultToken) + } else { + log.Entry().Debugf("Using Vaults AppRole Authentication") + client, err = vault.NewClientWithAppRole(clientConfig, creds.AppRoleID, creds.AppRoleSecretID) + } if err != nil { return nil, err } log.Entry().Infof("Fetching secrets from vault at %s", address) - return &client, nil + return client, nil } func resolveAllVaultReferences(config *StepConfig, client vaultClient, params []StepParameters) { for _, param := range params { - // we don't overwrite secrets that have already been set in any way - if _, ok := config.Config[param.Name].(string); ok { - continue - } if ref := param.GetReference("vaultSecret"); ref != nil { resolveVaultReference(ref, config, client, param) } @@ -81,6 +90,12 @@ func resolveAllVaultReferences(config *StepConfig, client vaultClient, params [] } func resolveVaultReference(ref *ResourceReference, config *StepConfig, client vaultClient, param StepParameters) { + vaultDisableOverwrite, _ := config.Config["vaultDisableOverwrite"].(bool) + if _, ok := config.Config[param.Name].(string); vaultDisableOverwrite && ok { + log.Entry().Debugf("Not fetching '%s' from vault since it has already been set", param.Name) + return + } + var secretValue *string for _, vaultPath := range ref.Paths { // it should be possible to configure the root path were the secret is stored diff --git a/pkg/config/vault_test.go b/pkg/config/vault_test.go index a08c0b927..d74a1b89f 100644 --- a/pkg/config/vault_test.go +++ b/pkg/config/vault_test.go @@ -28,6 +28,21 @@ func TestVaultConfigLoad(t *testing.T) { }) t.Run("Secrets are not overwritten", func(t *testing.T) { + vaultMock := &mocks.VaultMock{} + stepConfig := StepConfig{Config: map[string]interface{}{ + "vaultBasePath": "team1", + secretName: "preset value", + "vaultDisableOverwrite": true, + }} + stepParams := []StepParameters{stepParam(secretName, "vaultSecret", "$(vaultBasePath)/pipelineA")} + vaultData := map[string]string{secretName: "value1"} + vaultMock.On("GetKvSecret", "team1/pipelineA").Return(vaultData, nil) + resolveAllVaultReferences(&stepConfig, vaultMock, stepParams) + + assert.Equal(t, "preset value", stepConfig.Config[secretName]) + }) + + t.Run("Secrets can be overwritten", func(t *testing.T) { vaultMock := &mocks.VaultMock{} stepConfig := StepConfig{Config: map[string]interface{}{ "vaultBasePath": "team1", @@ -38,7 +53,7 @@ func TestVaultConfigLoad(t *testing.T) { vaultMock.On("GetKvSecret", "team1/pipelineA").Return(vaultData, nil) resolveAllVaultReferences(&stepConfig, vaultMock, stepParams) - assert.Equal(t, "preset value", stepConfig.Config[secretName]) + assert.Equal(t, "value1", stepConfig.Config[secretName]) }) t.Run("Error is passed through", func(t *testing.T) { diff --git a/pkg/vault/client.go b/pkg/vault/client.go index a460b4c7b..938660a25 100644 --- a/pkg/vault/client.go +++ b/pkg/vault/client.go @@ -46,6 +46,7 @@ func NewClient(config *Config, token string) (Client, error) { } client.SetToken(token) + log.Entry().Debugf("Login to vault %s in namespace %s successfull", config.Address, config.Namespace) return Client{client.Logical(), config}, nil } @@ -83,7 +84,6 @@ func NewClientWithAppRole(config *Config, roleID, secretID string) (Client, erro return Client{}, fmt.Errorf("Could not obtain token from approle with role_id %s", roleID) } - log.Entry().Debugf("Login to vault %s in namespace %s successfull", config.Address, config.Namespace) return NewClient(config, authInfo.ClientToken) } diff --git a/vars/piperExecuteBin.groovy b/vars/piperExecuteBin.groovy index b4abae3f9..0ea89f8f5 100644 --- a/vars/piperExecuteBin.groovy +++ b/vars/piperExecuteBin.groovy @@ -28,6 +28,7 @@ void call(Map parameters = [:], String stepName, String metadataFile, List crede prepareExecution(script, utils, parameters) prepareMetadataResource(script, metadataFile) Map stepParameters = prepareStepParameters(parameters) + echo "Step params $stepParameters" withEnv([ "PIPER_parametersJSON=${groovy.json.JsonOutput.toJson(stepParameters)}", @@ -155,10 +156,8 @@ void dockerWrapper(script, stepName, config, body) { // reused in sonarExecuteScan void credentialWrapper(config, List credentialInfo, body) { - if (config.containsKey('vaultAppRoleTokenCredentialsId') && config.containsKey('vaultAppRoleSecretTokenCredentialsId')) { - credentialInfo = [[type: 'token', id: 'vaultAppRoleTokenCredentialsId', env: ['PIPER_vaultAppRoleID']], - [type: 'token', id: 'vaultAppRoleSecretTokenCredentialsId', env: ['PIPER_vaultAppRoleSecretID']]] - } + credentialInfo = handleVaultCredentials(config, credentialInfo) + if (credentialInfo.size() > 0) { def creds = [] def sshCreds = [] @@ -170,7 +169,7 @@ void credentialWrapper(config, List credentialInfo, body) { credentialsId = config[cred.id] } if (credentialsId) { - switch(cred.type) { + switch (cred.type) { case "file": creds.add(file(credentialsId: credentialsId, variable: cred.env[0])) break @@ -184,11 +183,17 @@ void credentialWrapper(config, List credentialInfo, body) { sshCreds.add(credentialsId) break default: - error ("invalid credential type: ${cred.type}") + error("invalid credential type: ${cred.type}") } } } + // remove credentialIds that were probably defaulted and which are not present in jenkins + if (containsVaultConfig(config)) { + creds = removeMissingCredentials(creds, config) + sshCreds = removeMissingCredentials(sshCreds, config) + } + if (sshCreds.size() > 0) { sshagent (sshCreds) { withCredentials(creds) { @@ -205,6 +210,41 @@ void credentialWrapper(config, List credentialInfo, body) { } } +List removeMissingCredentials(List creds, Map config) { + return creds.findAll { credentialExists(it, config) } +} + +boolean credentialExists(cred, Map config) { + try { + withCredentials([cred]) { + return true + } + } catch (e) { + return false + } +} + +boolean containsVaultConfig(Map config) { + def approleIsUsed = config.containsKey('vaultAppRoleTokenCredentialsId') && config.containsKey('vaultAppRoleSecretTokenCredentialsId') + def tokenIsUsed = config.containsKey('vaultTokenCredentialsId') + + return approleIsUsed || tokenIsUsed +} + +// Injects vaultCredentials if steps supports resolving parameters from vault +List handleVaultCredentials(config, List credentialInfo) { + if (config.containsKey('vaultAppRoleTokenCredentialsId') && config.containsKey('vaultAppRoleSecretTokenCredentialsId')) { + credentialInfo += [[type: 'token', id: 'vaultAppRoleTokenCredentialsId', env: ['PIPER_vaultAppRoleID']], + [type: 'token', id: 'vaultAppRoleSecretTokenCredentialsId', env: ['PIPER_vaultAppRoleSecretID']]] + } + + if (config.containsKey('vaultTokenCredentialsId')) { + credentialInfo += [[type: 'token', id: 'vaultTokenCredentialsId', env: ['PIPER_vaultToken']]] + } + + return credentialInfo +} + // reused in sonarExecuteScan void handleErrorDetails(String stepName, Closure body) { try { @@ -218,7 +258,7 @@ void handleErrorDetails(String stepName, Closure body) { errorCategory = " (category: ${errorDetails.category})" DebugReport.instance.failedBuild.category = errorDetails.category } - error "[${stepName}] Step execution failed${errorCategory}. Error: ${errorDetails.error?:errorDetails.message}" + error "[${stepName}] Step execution failed${errorCategory}. Error: ${errorDetails.error ?: errorDetails.message}" } error "[${stepName}] Step execution failed. Error: ${ex}, please see log file for more details." }