From af5b738982a41458a1849634c2ee44173366aef2 Mon Sep 17 00:00:00 2001 From: Jordi van Liempt <35920075+jliempt@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:20:28 +0200 Subject: [PATCH] feat(trustengine): Integrate Trust Engine into step config resolver (#5032) * trust engine config and handelling for vault * add function for resolving trust engine reference * refactor * add basic test * adapt to new trust engine response format * remove accidental cyclic dependency * move trust engine hook config * refactor by separating code from vault * move trust engine files to own pkg * adapt to changes of previous commit * log full error response of trust engine API * enable getting multiple tokens from trustengine * remove comment * incorporate review comments * go generate * update unit tests * apply suggested changes from code review * fix unit tests * add unit tests for config pkg * make changes based on review comments * make trust engine token available in GeneralConfig and minor fixes * fix error logic when reading trust engine hook * make getResponse more flexible and update logging * update resource reference format * improve URL handling * improve logging * use errors.Wrap() instead of errors.Join() * update log messages based on suggestions * remove trustengine resource ref from Sonar step --------- Co-authored-by: Keshav Co-authored-by: jliempt <> --- cmd/piper.go | 19 +++- pkg/config/config.go | 32 ++++--- pkg/config/trustengine.go | 67 ++++++++++++++ pkg/config/trustengine_test.go | 74 +++++++++++++++ pkg/trustengine/trustengine.go | 135 ++++++++++++++++++++++++++++ pkg/trustengine/trustengine_test.go | 80 +++++++++++++++++ 6 files changed, 393 insertions(+), 14 deletions(-) create mode 100644 pkg/config/trustengine.go create mode 100644 pkg/config/trustengine_test.go create mode 100644 pkg/trustengine/trustengine.go create mode 100644 pkg/trustengine/trustengine_test.go diff --git a/cmd/piper.go b/cmd/piper.go index fa6f58536..65741c1d7 100644 --- a/cmd/piper.go +++ b/cmd/piper.go @@ -41,6 +41,7 @@ type GeneralConfigOptions struct { VaultServerURL string VaultNamespace string VaultPath string + TrustEngineToken string HookConfig HookConfiguration MetaDataResolver func() map[string]config.StepData GCPJsonKeyFilePath string @@ -51,10 +52,11 @@ type GeneralConfigOptions struct { // HookConfiguration contains the configuration for supported hooks, so far Sentry and Splunk are supported. type HookConfiguration struct { - SentryConfig SentryConfiguration `json:"sentry,omitempty"` - SplunkConfig SplunkConfiguration `json:"splunk,omitempty"` - PendoConfig PendoConfiguration `json:"pendo,omitempty"` - OIDCConfig OIDCConfiguration `json:"oidc,omitempty"` + SentryConfig SentryConfiguration `json:"sentry,omitempty"` + SplunkConfig SplunkConfiguration `json:"splunk,omitempty"` + PendoConfig PendoConfiguration `json:"pendo,omitempty"` + OIDCConfig OIDCConfiguration `json:"oidc,omitempty"` + TrustEngineConfig TrustEngineConfiguration `json:"trustengine,omitempty"` } // SentryConfiguration defines the configuration options for the Sentry logging system @@ -82,6 +84,12 @@ type OIDCConfiguration struct { RoleID string `json:",roleID,omitempty"` } +type TrustEngineConfiguration struct { + ServerURL string `json:"baseURL,omitempty"` + TokenEndPoint string `json:"tokenEndPoint,omitempty"` + TokenQueryParamName string `json:"tokenQueryParamName,omitempty"` +} + var rootCmd = &cobra.Command{ Use: "piper", Short: "Executes CI/CD steps from project 'Piper' ", @@ -369,6 +377,9 @@ func PrepareConfig(cmd *cobra.Command, metadata *config.StepData, stepName strin } myConfig.SetVaultCredentials(GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID, GeneralConfig.VaultToken) + GeneralConfig.TrustEngineToken = os.Getenv("PIPER_trustEngineToken") + myConfig.SetTrustEngineToken(GeneralConfig.TrustEngineToken) + if len(GeneralConfig.StepConfigJSON) != 0 { // ignore config & defaults in favor of passed stepConfigJSON stepConfig = config.GetStepConfigWithJSON(flagValues, GeneralConfig.StepConfigJSON, filters) diff --git a/pkg/config/config.go b/pkg/config/config.go index e41873f89..e35aa589e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -11,6 +11,8 @@ import ( "regexp" "strings" + "github.com/SAP/jenkins-library/pkg/trustengine" + piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" @@ -21,16 +23,17 @@ import ( // Config defines the structure of the config files type Config struct { - CustomDefaults []string `json:"customDefaults,omitempty"` - General map[string]interface{} `json:"general"` - Stages map[string]map[string]interface{} `json:"stages"` - Steps map[string]map[string]interface{} `json:"steps"` - Hooks map[string]interface{} `json:"hooks,omitempty"` - defaults PipelineDefaults - initialized bool - accessTokens map[string]string - openFile func(s string, t map[string]string) (io.ReadCloser, error) - vaultCredentials VaultCredentials + CustomDefaults []string `json:"customDefaults,omitempty"` + General map[string]interface{} `json:"general"` + Stages map[string]map[string]interface{} `json:"stages"` + Steps map[string]map[string]interface{} `json:"steps"` + Hooks map[string]interface{} `json:"hooks,omitempty"` + defaults PipelineDefaults + initialized bool + accessTokens map[string]string + openFile func(s string, t map[string]string) (io.ReadCloser, error) + vaultCredentials VaultCredentials + trustEngineConfiguration trustengine.Configuration } // StepConfig defines the structure for merged step configuration @@ -270,6 +273,15 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri } } + // hooks need to have been loaded from the defaults before the server URL is known + err = c.setTrustEngineConfiguration(stepConfig.HookConfig) + if err != nil { + log.Entry().WithError(err).Debug("Trust Engine lookup skipped due to missing or incorrect configuration") + } else { + trustengineClient := trustengine.PrepareClient(&piperhttp.Client{}, c.trustEngineConfiguration) + resolveAllTrustEngineReferences(&stepConfig, append(parameters, ReportingParameters.Parameters...), c.trustEngineConfiguration, trustengineClient) + } + // finally do the condition evaluation post processing for _, p := range parameters { if len(p.Conditions) > 0 { diff --git a/pkg/config/trustengine.go b/pkg/config/trustengine.go new file mode 100644 index 000000000..144804c18 --- /dev/null +++ b/pkg/config/trustengine.go @@ -0,0 +1,67 @@ +package config + +import ( + "errors" + + piperhttp "github.com/SAP/jenkins-library/pkg/http" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/trustengine" +) + +// const RefTypeTrustengineSecretFile = "trustengineSecretFile" +const RefTypeTrustengineSecret = "trustengineSecret" + +// resolveAllTrustEngineReferences retrieves all the step's secrets from the Trust Engine +func resolveAllTrustEngineReferences(config *StepConfig, params []StepParameters, trustEngineConfiguration trustengine.Configuration, client *piperhttp.Client) { + for _, param := range params { + if ref := param.GetReference(RefTypeTrustengineSecret); ref != nil { + if config.Config[param.Name] == "" { + log.Entry().Infof("Getting '%s' from Trust Engine", param.Name) + token, err := trustengine.GetToken(ref.Default, client, trustEngineConfiguration) + if err != nil { + log.Entry().Info(" failed") + log.Entry().WithError(err).Debugf("Couldn't get '%s' token from Trust Engine", ref.Default) + continue + } + log.RegisterSecret(token) + config.Config[param.Name] = token + log.Entry().Info(" succeeded") + } else { + log.Entry().Debugf("Skipping retrieval of '%s' from Trust Engine: parameter already set", param.Name) + } + } + } +} + +// setTrustEngineConfiguration sets the server URL for the Trust Engine by taking it from the hooks +func (c *Config) setTrustEngineConfiguration(hookConfig map[string]interface{}) error { + trustEngineHook, ok := hookConfig["trustengine"].(map[string]interface{}) + if !ok { + return errors.New("no Trust Engine hook configuration found") + } + if serverURL, ok := trustEngineHook["serverURL"].(string); ok { + c.trustEngineConfiguration.ServerURL = serverURL + } else { + return errors.New("no Trust Engine server URL found") + } + if tokenEndPoint, ok := trustEngineHook["tokenEndPoint"].(string); ok { + c.trustEngineConfiguration.TokenEndPoint = tokenEndPoint + } else { + return errors.New("no Trust Engine service endpoint found") + } + if tokenQueryParamName, ok := trustEngineHook["tokenQueryParamName"].(string); ok { + c.trustEngineConfiguration.TokenQueryParamName = tokenQueryParamName + } else { + return errors.New("no Trust Engine query parameter name found") + } + + if len(c.trustEngineConfiguration.Token) == 0 { + log.Entry().Debug("no Trust Engine token found") + } + return nil +} + +// SetTrustEngineToken sets the token for the Trust Engine +func (c *Config) SetTrustEngineToken(token string) { + c.trustEngineConfiguration.Token = token +} diff --git a/pkg/config/trustengine_test.go b/pkg/config/trustengine_test.go new file mode 100644 index 000000000..c475a9e8b --- /dev/null +++ b/pkg/config/trustengine_test.go @@ -0,0 +1,74 @@ +//go:build unit +// +build unit + +package config + +import ( + "fmt" + piperhttp "github.com/SAP/jenkins-library/pkg/http" + "github.com/SAP/jenkins-library/pkg/trustengine" + "github.com/jarcoal/httpmock" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +const secretName = "sonar" +const secretNameInTrustEngine = "sonarTrustengineSecretName" +const testServerURL = "https://www.project-piper.io" +const testTokenEndPoint = "tokens" +const testTokenQueryParamName = "systems" +const mockSonarToken = "mockSonarToken" + +var testFullURL = fmt.Sprintf("%s/%s?%s=", testServerURL, testTokenEndPoint, testTokenQueryParamName) +var mockSingleTokenResponse = fmt.Sprintf("{\"sonar\": \"%s\"}", mockSonarToken) + +func TestTrustEngineConfig(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + httpmock.RegisterResponder(http.MethodGet, testFullURL+"sonar", httpmock.NewStringResponder(200, mockSingleTokenResponse)) + + stepParams := []StepParameters{createStepParam(secretName, RefTypeTrustengineSecret, secretNameInTrustEngine, secretName)} + + var trustEngineConfiguration = trustengine.Configuration{ + Token: "testToken", + ServerURL: testServerURL, + TokenEndPoint: testTokenEndPoint, + TokenQueryParamName: testTokenQueryParamName, + } + client := &piperhttp.Client{} + client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true}) + + t.Run("Load secret from Trust Engine - secret not set yet by Vault or config.yml", func(t *testing.T) { + stepConfig := &StepConfig{Config: map[string]interface{}{ + secretName: "", + }} + + resolveAllTrustEngineReferences(stepConfig, stepParams, trustEngineConfiguration, client) + assert.Equal(t, mockSonarToken, stepConfig.Config[secretName]) + }) + + t.Run("Load secret from Trust Engine - secret already by Vault or config.yml", func(t *testing.T) { + stepConfig := &StepConfig{Config: map[string]interface{}{ + secretName: "aMockTokenFromVault", + }} + + resolveAllTrustEngineReferences(stepConfig, stepParams, trustEngineConfiguration, client) + assert.NotEqual(t, mockSonarToken, stepConfig.Config[secretName]) + }) +} + +func createStepParam(name, refType, trustengineSecretNameProperty, defaultSecretNameName string) StepParameters { + return StepParameters{ + Name: name, + Aliases: []Alias{}, + ResourceRef: []ResourceReference{ + { + Type: refType, + Name: trustengineSecretNameProperty, + Default: defaultSecretNameName, + }, + }, + } +} diff --git a/pkg/trustengine/trustengine.go b/pkg/trustengine/trustengine.go new file mode 100644 index 000000000..99caf500a --- /dev/null +++ b/pkg/trustengine/trustengine.go @@ -0,0 +1,135 @@ +package trustengine + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + piperhttp "github.com/SAP/jenkins-library/pkg/http" + "github.com/SAP/jenkins-library/pkg/log" +) + +type Secret struct { + Token string + System string +} + +type Response struct { + Secrets []Secret +} + +type Configuration struct { + ServerURL string + TokenEndPoint string + TokenQueryParamName string + Token string +} + +// GetToken requests a single token +func GetToken(refName string, client *piperhttp.Client, trustEngineConfiguration Configuration) (string, error) { + secrets, err := GetSecrets([]string{refName}, client, trustEngineConfiguration) + if err != nil { + return "", errors.Wrap(err, "couldn't get token from trust engine") + } + for _, s := range secrets { + if s.System == refName { + return s.Token, nil + } + } + return "", errors.New("could not find token in trust engine response") +} + +// GetSecrets transforms the trust engine JSON response into trust engine secrets, and can be used to request multiple tokens +func GetSecrets(refNames []string, client *piperhttp.Client, trustEngineConfiguration Configuration) ([]Secret, error) { + var secrets []Secret + query := url.Values{ + trustEngineConfiguration.TokenQueryParamName: { + strings.Join(refNames, ","), + }, + } + response, err := getResponse(trustEngineConfiguration.ServerURL, trustEngineConfiguration.TokenEndPoint, query, client) + if err != nil { + return secrets, errors.Wrap(err, "getting secrets from trust engine failed") + } + for k, v := range response { + secrets = append(secrets, Secret{ + System: k, + Token: v}) + } + + return secrets, nil +} + +// getResponse returns a map of the JSON response that the trust engine puts out +func getResponse(serverURL, endpoint string, query url.Values, client *piperhttp.Client) (map[string]string, error) { + var secrets map[string]string + + rawURL, err := parseURL(serverURL, endpoint, query) + if err != nil { + return secrets, errors.Wrap(err, "parsing trust engine url failed") + } + header := make(http.Header) + header.Add("Accept", "application/json") + + log.Entry().Debugf(" with URL %s", rawURL) + response, err := client.SendRequest(http.MethodGet, rawURL, nil, header, nil) + if err != nil { + if response != nil { + // the body contains full error message which we want to log + defer response.Body.Close() + bodyBytes, bodyErr := io.ReadAll(response.Body) + if bodyErr == nil { + err = errors.Wrap(err, string(bodyBytes)) + } + } + return secrets, errors.Wrap(err, "getting response from trust engine failed") + } + defer response.Body.Close() + + err = json.NewDecoder(response.Body).Decode(&secrets) + if err != nil { + return secrets, errors.Wrap(err, "getting response from trust engine failed") + } + + return secrets, nil +} + +// parseURL creates the full URL for a Trust Engine GET request +func parseURL(serverURL, endpoint string, query url.Values) (string, error) { + rawFullEndpoint, err := url.JoinPath(serverURL, endpoint) + if err != nil { + return "", errors.New("error parsing trust engine URL") + } + fullURL, err := url.Parse(rawFullEndpoint) + if err != nil { + return "", errors.New("error parsing trust engine URL") + } + // commas and spaces shouldn't be escaped since the Trust Engine won't accept it + unescapedRawQuery, err := url.QueryUnescape(query.Encode()) + if err != nil { + return "", errors.New("error parsing trust engine URL") + } + fullURL.RawQuery = unescapedRawQuery + return fullURL.String(), nil +} + +// PrepareClient adds the Trust Engine authentication token to the client +func PrepareClient(client *piperhttp.Client, trustEngineConfiguration Configuration) *piperhttp.Client { + var logEntry *logrus.Entry + if logrus.GetLevel() < logrus.DebugLevel { + logger := logrus.New() + logger.SetOutput(io.Discard) + logEntry = logrus.NewEntry(logger) + } + client.SetOptions(piperhttp.ClientOptions{ + Token: fmt.Sprintf("Bearer %s", trustEngineConfiguration.Token), + Logger: logEntry, + }) + return client +} diff --git a/pkg/trustengine/trustengine_test.go b/pkg/trustengine/trustengine_test.go new file mode 100644 index 000000000..d1ca1b179 --- /dev/null +++ b/pkg/trustengine/trustengine_test.go @@ -0,0 +1,80 @@ +//go:build unit +// +build unit + +package trustengine + +import ( + "fmt" + "github.com/jarcoal/httpmock" + "net/http" + "testing" + + piperhttp "github.com/SAP/jenkins-library/pkg/http" + "github.com/stretchr/testify/assert" +) + +const testServerURL = "https://www.project-piper.io" +const testTokenEndPoint = "tokens" +const testTokenQueryParamName = "systems" +const mockSonarToken = "mockSonarToken" +const mockblackduckToken = "mockblackduckToken" +const errorMsg403 = "unauthorized to request token" + +var testFullURL = fmt.Sprintf("%s/%s?%s=", testServerURL, testTokenEndPoint, testTokenQueryParamName) +var mockSingleTokenResponse = fmt.Sprintf("{\"sonar\": \"%s\"}", mockSonarToken) +var mockTwoTokensResponse = fmt.Sprintf("{\"sonar\": \"%s\", \"blackduck\": \"%s\"}", mockSonarToken, mockblackduckToken) +var trustEngineConfiguration = Configuration{ + Token: "testToken", + ServerURL: testServerURL, + TokenEndPoint: testTokenEndPoint, + TokenQueryParamName: testTokenQueryParamName, +} + +func TestTrustEngine(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + t.Run("Get Sonar token - happy path", func(t *testing.T) { + httpmock.RegisterResponder(http.MethodGet, testFullURL+"sonar", httpmock.NewStringResponder(200, mockSingleTokenResponse)) + + client := &piperhttp.Client{} + client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true}) + + token, err := GetToken("sonar", client, trustEngineConfiguration) + assert.NoError(t, err) + assert.Equal(t, mockSonarToken, token) + }) + + t.Run("Get multiple tokens - happy path", func(t *testing.T) { + httpmock.RegisterResponder(http.MethodGet, testFullURL+"sonar,blackduck", httpmock.NewStringResponder(200, mockTwoTokensResponse)) + + client := &piperhttp.Client{} + client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true}) + + secrets, err := GetSecrets([]string{"sonar", "blackduck"}, client, trustEngineConfiguration) + + assert.NoError(t, err) + assert.Len(t, secrets, 2) + for _, s := range secrets { + switch system := s.System; system { + case "sonar": + assert.Equal(t, mockSonarToken, s.Token) + case "blackduck": + assert.Equal(t, mockblackduckToken, s.Token) + default: + continue + } + } + }) + + t.Run("Get Sonar token - 403 error", func(t *testing.T) { + httpmock.RegisterResponder(http.MethodGet, testFullURL+"sonar", httpmock.NewStringResponder(403, errorMsg403)) + + client := &piperhttp.Client{} + client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true}) + + _, err := GetToken("sonar", client, trustEngineConfiguration) + assert.Error(t, err) + }) + +}