From 27c3c3c4c7a1e82271a7a8fab558d59e5ff601fd Mon Sep 17 00:00:00 2001 From: Jk1484 <35270240+Jk1484@users.noreply.github.com> Date: Mon, 22 May 2023 13:49:28 +0500 Subject: [PATCH] feat(vault): support for multiple general purpose credential paths (#4360) * created wrapper * tests added * update documentation * tests data race fix --------- Co-authored-by: Jordi van Liempt <35920075+jliempt@users.noreply.github.com> --- documentation/docs/infrastructure/vault.md | 12 ++ pkg/config/config.go | 4 +- pkg/config/vault.go | 46 ++++++++ pkg/config/vault_test.go | 127 ++++++++++++++++++--- 4 files changed, 170 insertions(+), 19 deletions(-) diff --git a/documentation/docs/infrastructure/vault.md b/documentation/docs/infrastructure/vault.md index a780f57c7..d58a59259 100644 --- a/documentation/docs/infrastructure/vault.md +++ b/documentation/docs/infrastructure/vault.md @@ -109,6 +109,18 @@ steps: vaultCredentialKeys: ['myAppId', 'myAppSecret'] ``` +In case if you want to retrieve secrets from multiple vault folders, pass several paths with keys: + +```yaml +general: + < your Vault configuration > # see above +... +steps: + < piper go step >: + vaultCredentialPath: ['myStepCredentials1', 'myStepCredentials2'] + vaultCredentialKeys: [['myAppId1', 'myAppSecret1'], ['myAppId2', 'myAppSecret2']] +``` + The `vaultCredentialPath` parameter is the endpoint of your credential path in Vault. Depending on your _general_ config, the lookup for the credential IDs will be done in the following order respectively locations. The first path with found general purpose credentials will be used. 1. `/` diff --git a/pkg/config/config.go b/pkg/config/config.go index 7133b72e3..fcbb0988f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -265,8 +265,8 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri if vaultClient != nil { defer vaultClient.MustRevokeToken() resolveAllVaultReferences(&stepConfig, vaultClient, append(parameters, ReportingParameters.Parameters...)) - resolveVaultTestCredentials(&stepConfig, vaultClient) - resolveVaultCredentials(&stepConfig, vaultClient) + resolveVaultTestCredentialsWrapper(&stepConfig, vaultClient) + resolveVaultCredentialsWrapper(&stepConfig, vaultClient) } } diff --git a/pkg/config/vault.go b/pkg/config/vault.go index 298c87c80..70525ab29 100644 --- a/pkg/config/vault.go +++ b/pkg/config/vault.go @@ -172,6 +172,52 @@ func resolveVaultReference(ref *ResourceReference, config *StepConfig, client va } } +func resolveVaultTestCredentialsWrapper(config *StepConfig, client vaultClient) { + log.Entry().Debugln("resolveVaultTestCredentialsWrapper") + resolveVaultTestCredentialsWrapperBase(config, client, vaultTestCredentialPath, vaultTestCredentialKeys, resolveVaultTestCredentials) +} + +func resolveVaultCredentialsWrapper(config *StepConfig, client vaultClient) { + log.Entry().Debugln("resolveVaultCredentialsWrapper") + resolveVaultTestCredentialsWrapperBase(config, client, vaultCredentialPath, vaultCredentialKeys, resolveVaultCredentials) +} + +func resolveVaultTestCredentialsWrapperBase( + config *StepConfig, client vaultClient, + vaultCredPath, vaultCredKeys string, + resolveVaultCredentials func(config *StepConfig, client vaultClient), +) { + switch config.Config[vaultCredPath].(type) { + case string: + resolveVaultCredentials(config, client) + case []interface{}: + vaultCredentialPathCopy := config.Config[vaultCredPath] + vaultCredentialKeysCopy := config.Config[vaultCredKeys] + + if _, ok := vaultCredentialKeysCopy.([]interface{}); !ok { + log.Entry().Debugf("Not fetching credentials from vault since they are not (properly) configured: unknown type of keys") + return + } + + if len(vaultCredentialKeysCopy.([]interface{})) != len(vaultCredentialPathCopy.([]interface{})) { + log.Entry().Debugf("Not fetching credentials from vault since they are not (properly) configured: not same count of values and keys") + return + } + + for i := 0; i < len(vaultCredentialPathCopy.([]interface{})); i++ { + config.Config[vaultCredPath] = vaultCredentialPathCopy.([]interface{})[i] + config.Config[vaultCredKeys] = vaultCredentialKeysCopy.([]interface{})[i] + resolveVaultCredentials(config, client) + } + + config.Config[vaultCredPath] = vaultCredentialPathCopy + config.Config[vaultCredKeys] = vaultCredentialKeysCopy + default: + log.Entry().Debugf("Not fetching credentials from vault since they are not (properly) configured: unknown type of path") + return + } +} + // resolve test credential keys and expose as environment variables func resolveVaultTestCredentials(config *StepConfig, client vaultClient) { credPath, pathOk := config.Config[vaultTestCredentialPath].(string) diff --git a/pkg/config/vault_test.go b/pkg/config/vault_test.go index bc298e63c..f2c4b8441 100644 --- a/pkg/config/vault_test.go +++ b/pkg/config/vault_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "path" + "strconv" "strings" "testing" @@ -227,6 +228,93 @@ func addAlias(param *StepParameters, aliasName string) { param.Aliases = append(param.Aliases, alias) } +func TestResolveVaultTestCredentialsWrapper(t *testing.T) { + t.Parallel() + t.Run("Default test credential prefix", func(t *testing.T) { + t.Parallel() + // init + vaultMock := &mocks.VaultMock{} + envPrefix := "PIPER_TESTCREDENTIAL_" + stepConfig := StepConfig{Config: map[string]interface{}{ + "vaultPath": "team1", + "vaultTestCredentialPath": []interface{}{"appCredentials1", "appCredentials2"}, + "vaultTestCredentialKeys": []interface{}{[]interface{}{"appUser1", "appUserPw1"}, []interface{}{"appUser2", "appUserPw2"}}, + }} + + defer os.Unsetenv("PIPER_TESTCREDENTIAL_APPUSER1") + defer os.Unsetenv("PIPER_TESTCREDENTIAL_APPUSERPW1") + defer os.Unsetenv("PIPER_TESTCREDENTIAL_APPUSER2") + defer os.Unsetenv("PIPER_TESTCREDENTIAL_APPUSERPW2") + + // mock + vaultData1 := map[string]string{"appUser1": "test-user", "appUserPw1": "password1234"} + vaultMock.On("GetKvSecret", "team1/appCredentials1").Return(vaultData1, nil) + vaultData2 := map[string]string{"appUser2": "test-user", "appUserPw2": "password1234"} + vaultMock.On("GetKvSecret", "team1/appCredentials2").Return(vaultData2, nil) + + // test + resolveVaultTestCredentialsWrapper(&stepConfig, vaultMock) + + // assert + for k, expectedValue := range vaultData1 { + env := envPrefix + strings.ToUpper(k) + assert.NotEmpty(t, os.Getenv(env)) + assert.Equal(t, expectedValue, os.Getenv(env)) + } + + // assert + for k, expectedValue := range vaultData2 { + env := envPrefix + strings.ToUpper(k) + assert.NotEmpty(t, os.Getenv(env)) + assert.Equal(t, expectedValue, os.Getenv(env)) + } + }) + + // Test empty and non-empty custom general purpose credential prefix + envPrefixes := []string{"CUSTOM_MYCRED1_", ""} + for idx, envPrefix := range envPrefixes { + tEnvPrefix := envPrefix + // this variable is used to avoid race condition, because tests are running in parallel + // env variable with default prefix is being created for each iteration and being set and unset asynchronously + // race condition may occur while one function sets and tries to assert if it exists but the other unsets it before it + stIdx := strconv.Itoa(idx) + t.Run("Custom general purpose credential prefix along with fixed standard prefix", func(t *testing.T) { + t.Parallel() + // init + vaultMock := &mocks.VaultMock{} + standardEnvPrefix := "PIPER_VAULTCREDENTIAL_" + stepConfig := StepConfig{Config: map[string]interface{}{ + "vaultPath": "team1", + "vaultCredentialPath": "appCredentials3", + "vaultCredentialKeys": []interface{}{"appUser3" + stIdx, "appUserPw3" + stIdx}, + "vaultCredentialEnvPrefix": tEnvPrefix, + }} + + defer os.Unsetenv(tEnvPrefix + "APPUSER3" + stIdx) + defer os.Unsetenv(tEnvPrefix + "APPUSERPW3" + stIdx) + defer os.Unsetenv("PIPER_VAULTCREDENTIAL_APPUSER3" + stIdx) + defer os.Unsetenv("PIPER_VAULTCREDENTIAL_APPUSERPW3" + stIdx) + + // mock + vaultData := map[string]string{"appUser3" + stIdx: "test-user", "appUserPw3" + stIdx: "password1234"} + vaultMock.On("GetKvSecret", "team1/appCredentials3").Return(vaultData, nil) + + // test + resolveVaultCredentialsWrapper(&stepConfig, vaultMock) + + // assert + for k, expectedValue := range vaultData { + env := tEnvPrefix + strings.ToUpper(k) + assert.NotEmpty(t, os.Getenv(env)) + assert.Equal(t, expectedValue, os.Getenv(env)) + standardEnv := standardEnvPrefix + strings.ToUpper(k) + assert.NotEmpty(t, os.Getenv(standardEnv)) + assert.Equal(t, expectedValue, os.Getenv(standardEnv)) + } + }) + } +} + func TestResolveVaultTestCredentials(t *testing.T) { t.Parallel() t.Run("Default test credential prefix", func(t *testing.T) { @@ -237,14 +325,14 @@ func TestResolveVaultTestCredentials(t *testing.T) { stepConfig := StepConfig{Config: map[string]interface{}{ "vaultPath": "team1", "vaultTestCredentialPath": "appCredentials", - "vaultTestCredentialKeys": []interface{}{"appUser", "appUserPw"}, + "vaultTestCredentialKeys": []interface{}{"appUser4", "appUserPw4"}, }} - defer os.Unsetenv("PIPER_TESTCREDENTIAL_APPUSER") - defer os.Unsetenv("PIPER_TESTCREDENTIAL_APPUSERPW") + defer os.Unsetenv("PIPER_TESTCREDENTIAL_APPUSER4") + defer os.Unsetenv("PIPER_TESTCREDENTIAL_APPUSERPW4") // mock - vaultData := map[string]string{"appUser": "test-user", "appUserPw": "password1234"} + vaultData := map[string]string{"appUser4": "test-user", "appUserPw4": "password1234"} vaultMock.On("GetKvSecret", "team1/appCredentials").Return(vaultData, nil) // test @@ -260,7 +348,12 @@ func TestResolveVaultTestCredentials(t *testing.T) { // Test empty and non-empty custom general purpose credential prefix envPrefixes := []string{"CUSTOM_MYCRED_", ""} - for _, envPrefix := range envPrefixes { + for idx, envPrefix := range envPrefixes { + tEnvPrefix := envPrefix + // this variable is used to avoid race condition, because tests are running in parallel + // env variable with default prefix is being created for each iteration and being set and unset asynchronously + // race condition may occur while one function sets and tries to assert if it exists but the other unsets it before it + stIdx := strconv.Itoa(idx) t.Run("Custom general purpose credential prefix along with fixed standard prefix", func(t *testing.T) { t.Parallel() // init @@ -269,17 +362,17 @@ func TestResolveVaultTestCredentials(t *testing.T) { stepConfig := StepConfig{Config: map[string]interface{}{ "vaultPath": "team1", "vaultCredentialPath": "appCredentials", - "vaultCredentialKeys": []interface{}{"appUser", "appUserPw"}, - "vaultCredentialEnvPrefix": envPrefix, + "vaultCredentialKeys": []interface{}{"appUser5" + stIdx, "appUserPw5" + stIdx}, + "vaultCredentialEnvPrefix": tEnvPrefix, }} - defer os.Unsetenv(envPrefix + "APPUSER") - defer os.Unsetenv(envPrefix + "APPUSERPW") - defer os.Unsetenv("PIPER_VAULTCREDENTIAL_APPUSER") - defer os.Unsetenv("PIPER_VAULTCREDENTIAL_APPUSERPW") + defer os.Unsetenv(tEnvPrefix + "APPUSER5" + stIdx) + defer os.Unsetenv(tEnvPrefix + "APPUSERPW5" + stIdx) + defer os.Unsetenv("PIPER_VAULTCREDENTIAL_APPUSER5" + stIdx) + defer os.Unsetenv("PIPER_VAULTCREDENTIAL_APPUSERPW5" + stIdx) // mock - vaultData := map[string]string{"appUser": "test-user", "appUserPw": "password1234"} + vaultData := map[string]string{"appUser5" + stIdx: "test-user", "appUserPw5" + stIdx: "password1234"} vaultMock.On("GetKvSecret", "team1/appCredentials").Return(vaultData, nil) // test @@ -287,7 +380,7 @@ func TestResolveVaultTestCredentials(t *testing.T) { // assert for k, expectedValue := range vaultData { - env := envPrefix + strings.ToUpper(k) + env := tEnvPrefix + strings.ToUpper(k) assert.NotEmpty(t, os.Getenv(env)) assert.Equal(t, expectedValue, os.Getenv(env)) standardEnv := standardEnvPrefix + strings.ToUpper(k) @@ -305,15 +398,15 @@ func TestResolveVaultTestCredentials(t *testing.T) { stepConfig := StepConfig{Config: map[string]interface{}{ "vaultPath": "team1", "vaultTestCredentialPath": "appCredentials", - "vaultTestCredentialKeys": []interface{}{"appUser", "appUserPw"}, + "vaultTestCredentialKeys": []interface{}{"appUser6", "appUserPw6"}, "vaultTestCredentialEnvPrefix": envPrefix, }} - defer os.Unsetenv("CUSTOM_CREDENTIAL_APPUSER") - defer os.Unsetenv("CUSTOM_CREDENTIAL_APPUSERPW") + defer os.Unsetenv("CUSTOM_CREDENTIAL_APPUSER6") + defer os.Unsetenv("CUSTOM_CREDENTIAL_APPUSERPW6") // mock - vaultData := map[string]string{"appUser": "test-user", "appUserPw": "password1234"} + vaultData := map[string]string{"appUser6": "test-user", "appUserPw6": "password1234"} vaultMock.On("GetKvSecret", "team1/appCredentials").Return(vaultData, nil) // test