feat(trustengine): Integrate Trust Engine into step config resolver (#5032)

* trust engine config and handelling for vault * add function for resolving trust engine reference * refactor * add basic test * adapt to new trust engine response format * remove accidental cyclic dependency * move trust engine hook config * refactor by separating code from vault * move trust engine files to own pkg * adapt to changes of previous commit * log full error response of trust engine API * enable getting multiple tokens from trustengine * remove comment * incorporate review comments * go generate * update unit tests * apply suggested changes from code review * fix unit tests * add unit tests for config pkg * make changes based on review comments * make trust engine token available in GeneralConfig and minor fixes * fix error logic when reading trust engine hook * make getResponse more flexible and update logging * update resource reference format * improve URL handling * improve logging * use errors.Wrap() instead of errors.Join() * update log messages based on suggestions * remove trustengine resource ref from Sonar step --------- Co-authored-by: Keshav <anil.keshav@sap.com> Co-authored-by: jliempt <>
2025-02-21 19:48:53 +02:00 · 2024-09-11 15:20:28 +02:00 · 2024-09-11 15:20:28 +02:00 · af5b738982
commit af5b738982
parent 7e2604ad9e
6 changed files with 393 additions and 14 deletions
--- a/cmd/piper.go
+++ b/cmd/piper.go
@ -41,6 +41,7 @@ type GeneralConfigOptions struct {
 	VaultServerURL       string
 	VaultNamespace       string
 	VaultPath            string
+	TrustEngineToken     string
 	HookConfig           HookConfiguration
 	MetaDataResolver     func() map[string]config.StepData
 	GCPJsonKeyFilePath   string
@ -51,10 +52,11 @@ type GeneralConfigOptions struct {

 // HookConfiguration contains the configuration for supported hooks, so far Sentry and Splunk are supported.
 type HookConfiguration struct {
-	SentryConfig SentryConfiguration `json:"sentry,omitempty"`
-	SplunkConfig SplunkConfiguration `json:"splunk,omitempty"`
-	PendoConfig  PendoConfiguration  `json:"pendo,omitempty"`
-	OIDCConfig   OIDCConfiguration   `json:"oidc,omitempty"`
+	SentryConfig      SentryConfiguration      `json:"sentry,omitempty"`
+	SplunkConfig      SplunkConfiguration      `json:"splunk,omitempty"`
+	PendoConfig       PendoConfiguration       `json:"pendo,omitempty"`
+	OIDCConfig        OIDCConfiguration        `json:"oidc,omitempty"`
+	TrustEngineConfig TrustEngineConfiguration `json:"trustengine,omitempty"`
 }

 // SentryConfiguration defines the configuration options for the Sentry logging system
@ -82,6 +84,12 @@ type OIDCConfiguration struct {
 	RoleID string `json:",roleID,omitempty"`
 }

+type TrustEngineConfiguration struct {
+	ServerURL           string `json:"baseURL,omitempty"`
+	TokenEndPoint       string `json:"tokenEndPoint,omitempty"`
+	TokenQueryParamName string `json:"tokenQueryParamName,omitempty"`
+}
+
 var rootCmd = &cobra.Command{
 	Use:   "piper",
 	Short: "Executes CI/CD steps from project 'Piper' ",
@ -369,6 +377,9 @@ func PrepareConfig(cmd *cobra.Command, metadata *config.StepData, stepName strin
 	}
 	myConfig.SetVaultCredentials(GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID, GeneralConfig.VaultToken)

+	GeneralConfig.TrustEngineToken = os.Getenv("PIPER_trustEngineToken")
+	myConfig.SetTrustEngineToken(GeneralConfig.TrustEngineToken)
+
 	if len(GeneralConfig.StepConfigJSON) != 0 {
 		// ignore config & defaults in favor of passed stepConfigJSON
 		stepConfig = config.GetStepConfigWithJSON(flagValues, GeneralConfig.StepConfigJSON, filters)
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@ -11,6 +11,8 @@ import (
 	"regexp"
 	"strings"

+	"github.com/SAP/jenkins-library/pkg/trustengine"
+
 	piperhttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/SAP/jenkins-library/pkg/log"

@ -21,16 +23,17 @@ import (

 // Config defines the structure of the config files
 type Config struct {
-	CustomDefaults   []string                          `json:"customDefaults,omitempty"`
-	General          map[string]interface{}            `json:"general"`
-	Stages           map[string]map[string]interface{} `json:"stages"`
-	Steps            map[string]map[string]interface{} `json:"steps"`
-	Hooks            map[string]interface{}            `json:"hooks,omitempty"`
-	defaults         PipelineDefaults
-	initialized      bool
-	accessTokens     map[string]string
-	openFile         func(s string, t map[string]string) (io.ReadCloser, error)
-	vaultCredentials VaultCredentials
+	CustomDefaults           []string                          `json:"customDefaults,omitempty"`
+	General                  map[string]interface{}            `json:"general"`
+	Stages                   map[string]map[string]interface{} `json:"stages"`
+	Steps                    map[string]map[string]interface{} `json:"steps"`
+	Hooks                    map[string]interface{}            `json:"hooks,omitempty"`
+	defaults                 PipelineDefaults
+	initialized              bool
+	accessTokens             map[string]string
+	openFile                 func(s string, t map[string]string) (io.ReadCloser, error)
+	vaultCredentials         VaultCredentials
+	trustEngineConfiguration trustengine.Configuration
 }

 // StepConfig defines the structure for merged step configuration
@ -270,6 +273,15 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri
 		}
 	}

+	// hooks need to have been loaded from the defaults before the server URL is known
+	err = c.setTrustEngineConfiguration(stepConfig.HookConfig)
+	if err != nil {
+		log.Entry().WithError(err).Debug("Trust Engine lookup skipped due to missing or incorrect configuration")
+	} else {
+		trustengineClient := trustengine.PrepareClient(&piperhttp.Client{}, c.trustEngineConfiguration)
+		resolveAllTrustEngineReferences(&stepConfig, append(parameters, ReportingParameters.Parameters...), c.trustEngineConfiguration, trustengineClient)
+	}
+
 	// finally do the condition evaluation post processing
 	for _, p := range parameters {
 		if len(p.Conditions) > 0 {
--- a/pkg/config/trustengine.go
+++ b/pkg/config/trustengine.go
@ -0,0 +1,67 @@
+package config
+
+import (
+	"errors"
+
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/trustengine"
+)
+
+// const RefTypeTrustengineSecretFile = "trustengineSecretFile"
+const RefTypeTrustengineSecret = "trustengineSecret"
+
+// resolveAllTrustEngineReferences retrieves all the step's secrets from the Trust Engine
+func resolveAllTrustEngineReferences(config *StepConfig, params []StepParameters, trustEngineConfiguration trustengine.Configuration, client *piperhttp.Client) {
+	for _, param := range params {
+		if ref := param.GetReference(RefTypeTrustengineSecret); ref != nil {
+			if config.Config[param.Name] == "" {
+				log.Entry().Infof("Getting '%s' from Trust Engine", param.Name)
+				token, err := trustengine.GetToken(ref.Default, client, trustEngineConfiguration)
+				if err != nil {
+					log.Entry().Info(" failed")
+					log.Entry().WithError(err).Debugf("Couldn't get '%s' token from Trust Engine", ref.Default)
+					continue
+				}
+				log.RegisterSecret(token)
+				config.Config[param.Name] = token
+				log.Entry().Info(" succeeded")
+			} else {
+				log.Entry().Debugf("Skipping retrieval of '%s' from Trust Engine: parameter already set", param.Name)
+			}
+		}
+	}
+}
+
+// setTrustEngineConfiguration sets the server URL for the Trust Engine by taking it from the hooks
+func (c *Config) setTrustEngineConfiguration(hookConfig map[string]interface{}) error {
+	trustEngineHook, ok := hookConfig["trustengine"].(map[string]interface{})
+	if !ok {
+		return errors.New("no Trust Engine hook configuration found")
+	}
+	if serverURL, ok := trustEngineHook["serverURL"].(string); ok {
+		c.trustEngineConfiguration.ServerURL = serverURL
+	} else {
+		return errors.New("no Trust Engine server URL found")
+	}
+	if tokenEndPoint, ok := trustEngineHook["tokenEndPoint"].(string); ok {
+		c.trustEngineConfiguration.TokenEndPoint = tokenEndPoint
+	} else {
+		return errors.New("no Trust Engine service endpoint found")
+	}
+	if tokenQueryParamName, ok := trustEngineHook["tokenQueryParamName"].(string); ok {
+		c.trustEngineConfiguration.TokenQueryParamName = tokenQueryParamName
+	} else {
+		return errors.New("no Trust Engine query parameter name found")
+	}
+
+	if len(c.trustEngineConfiguration.Token) == 0 {
+		log.Entry().Debug("no Trust Engine token found")
+	}
+	return nil
+}
+
+// SetTrustEngineToken sets the token for the Trust Engine
+func (c *Config) SetTrustEngineToken(token string) {
+	c.trustEngineConfiguration.Token = token
+}
--- a/pkg/config/trustengine_test.go
+++ b/pkg/config/trustengine_test.go
@ -0,0 +1,74 @@
+//go:build unit
+// +build unit
+
+package config
+
+import (
+	"fmt"
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/SAP/jenkins-library/pkg/trustengine"
+	"github.com/jarcoal/httpmock"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const secretName = "sonar"
+const secretNameInTrustEngine = "sonarTrustengineSecretName"
+const testServerURL = "https://www.project-piper.io"
+const testTokenEndPoint = "tokens"
+const testTokenQueryParamName = "systems"
+const mockSonarToken = "mockSonarToken"
+
+var testFullURL = fmt.Sprintf("%s/%s?%s=", testServerURL, testTokenEndPoint, testTokenQueryParamName)
+var mockSingleTokenResponse = fmt.Sprintf("{\"sonar\": \"%s\"}", mockSonarToken)
+
+func TestTrustEngineConfig(t *testing.T) {
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+	httpmock.RegisterResponder(http.MethodGet, testFullURL+"sonar", httpmock.NewStringResponder(200, mockSingleTokenResponse))
+
+	stepParams := []StepParameters{createStepParam(secretName, RefTypeTrustengineSecret, secretNameInTrustEngine, secretName)}
+
+	var trustEngineConfiguration = trustengine.Configuration{
+		Token:               "testToken",
+		ServerURL:           testServerURL,
+		TokenEndPoint:       testTokenEndPoint,
+		TokenQueryParamName: testTokenQueryParamName,
+	}
+	client := &piperhttp.Client{}
+	client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true})
+
+	t.Run("Load secret from Trust Engine - secret not set yet by Vault or config.yml", func(t *testing.T) {
+		stepConfig := &StepConfig{Config: map[string]interface{}{
+			secretName: "",
+		}}
+
+		resolveAllTrustEngineReferences(stepConfig, stepParams, trustEngineConfiguration, client)
+		assert.Equal(t, mockSonarToken, stepConfig.Config[secretName])
+	})
+
+	t.Run("Load secret from Trust Engine - secret already by Vault or config.yml", func(t *testing.T) {
+		stepConfig := &StepConfig{Config: map[string]interface{}{
+			secretName: "aMockTokenFromVault",
+		}}
+
+		resolveAllTrustEngineReferences(stepConfig, stepParams, trustEngineConfiguration, client)
+		assert.NotEqual(t, mockSonarToken, stepConfig.Config[secretName])
+	})
+}
+
+func createStepParam(name, refType, trustengineSecretNameProperty, defaultSecretNameName string) StepParameters {
+	return StepParameters{
+		Name:    name,
+		Aliases: []Alias{},
+		ResourceRef: []ResourceReference{
+			{
+				Type:    refType,
+				Name:    trustengineSecretNameProperty,
+				Default: defaultSecretNameName,
+			},
+		},
+	}
+}
--- a/pkg/trustengine/trustengine.go
+++ b/pkg/trustengine/trustengine.go
@ -0,0 +1,135 @@
+package trustengine
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/SAP/jenkins-library/pkg/log"
+)
+
+type Secret struct {
+	Token  string
+	System string
+}
+
+type Response struct {
+	Secrets []Secret
+}
+
+type Configuration struct {
+	ServerURL           string
+	TokenEndPoint       string
+	TokenQueryParamName string
+	Token               string
+}
+
+// GetToken requests a single token
+func GetToken(refName string, client *piperhttp.Client, trustEngineConfiguration Configuration) (string, error) {
+	secrets, err := GetSecrets([]string{refName}, client, trustEngineConfiguration)
+	if err != nil {
+		return "", errors.Wrap(err, "couldn't get token from trust engine")
+	}
+	for _, s := range secrets {
+		if s.System == refName {
+			return s.Token, nil
+		}
+	}
+	return "", errors.New("could not find token in trust engine response")
+}
+
+// GetSecrets transforms the trust engine JSON response into trust engine secrets, and can be used to request multiple tokens
+func GetSecrets(refNames []string, client *piperhttp.Client, trustEngineConfiguration Configuration) ([]Secret, error) {
+	var secrets []Secret
+	query := url.Values{
+		trustEngineConfiguration.TokenQueryParamName: {
+			strings.Join(refNames, ","),
+		},
+	}
+	response, err := getResponse(trustEngineConfiguration.ServerURL, trustEngineConfiguration.TokenEndPoint, query, client)
+	if err != nil {
+		return secrets, errors.Wrap(err, "getting secrets from trust engine failed")
+	}
+	for k, v := range response {
+		secrets = append(secrets, Secret{
+			System: k,
+			Token:  v})
+	}
+
+	return secrets, nil
+}
+
+// getResponse returns a map of the JSON response that the trust engine puts out
+func getResponse(serverURL, endpoint string, query url.Values, client *piperhttp.Client) (map[string]string, error) {
+	var secrets map[string]string
+
+	rawURL, err := parseURL(serverURL, endpoint, query)
+	if err != nil {
+		return secrets, errors.Wrap(err, "parsing trust engine url failed")
+	}
+	header := make(http.Header)
+	header.Add("Accept", "application/json")
+
+	log.Entry().Debugf("  with URL %s", rawURL)
+	response, err := client.SendRequest(http.MethodGet, rawURL, nil, header, nil)
+	if err != nil {
+		if response != nil {
+			// the body contains full error message which we want to log
+			defer response.Body.Close()
+			bodyBytes, bodyErr := io.ReadAll(response.Body)
+			if bodyErr == nil {
+				err = errors.Wrap(err, string(bodyBytes))
+			}
+		}
+		return secrets, errors.Wrap(err, "getting response from trust engine failed")
+	}
+	defer response.Body.Close()
+
+	err = json.NewDecoder(response.Body).Decode(&secrets)
+	if err != nil {
+		return secrets, errors.Wrap(err, "getting response from trust engine failed")
+	}
+
+	return secrets, nil
+}
+
+// parseURL creates the full URL for a Trust Engine GET request
+func parseURL(serverURL, endpoint string, query url.Values) (string, error) {
+	rawFullEndpoint, err := url.JoinPath(serverURL, endpoint)
+	if err != nil {
+		return "", errors.New("error parsing trust engine URL")
+	}
+	fullURL, err := url.Parse(rawFullEndpoint)
+	if err != nil {
+		return "", errors.New("error parsing trust engine URL")
+	}
+	// commas and spaces shouldn't be escaped since the Trust Engine won't accept it
+	unescapedRawQuery, err := url.QueryUnescape(query.Encode())
+	if err != nil {
+		return "", errors.New("error parsing trust engine URL")
+	}
+	fullURL.RawQuery = unescapedRawQuery
+	return fullURL.String(), nil
+}
+
+// PrepareClient adds the Trust Engine authentication token to the client
+func PrepareClient(client *piperhttp.Client, trustEngineConfiguration Configuration) *piperhttp.Client {
+	var logEntry *logrus.Entry
+	if logrus.GetLevel() < logrus.DebugLevel {
+		logger := logrus.New()
+		logger.SetOutput(io.Discard)
+		logEntry = logrus.NewEntry(logger)
+	}
+	client.SetOptions(piperhttp.ClientOptions{
+		Token:  fmt.Sprintf("Bearer %s", trustEngineConfiguration.Token),
+		Logger: logEntry,
+	})
+	return client
+}
--- a/pkg/trustengine/trustengine_test.go
+++ b/pkg/trustengine/trustengine_test.go
@ -0,0 +1,80 @@
+//go:build unit
+// +build unit
+
+package trustengine
+
+import (
+	"fmt"
+	"github.com/jarcoal/httpmock"
+	"net/http"
+	"testing"
+
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/stretchr/testify/assert"
+)
+
+const testServerURL = "https://www.project-piper.io"
+const testTokenEndPoint = "tokens"
+const testTokenQueryParamName = "systems"
+const mockSonarToken = "mockSonarToken"
+const mockblackduckToken = "mockblackduckToken"
+const errorMsg403 = "unauthorized to request token"
+
+var testFullURL = fmt.Sprintf("%s/%s?%s=", testServerURL, testTokenEndPoint, testTokenQueryParamName)
+var mockSingleTokenResponse = fmt.Sprintf("{\"sonar\": \"%s\"}", mockSonarToken)
+var mockTwoTokensResponse = fmt.Sprintf("{\"sonar\": \"%s\", \"blackduck\": \"%s\"}", mockSonarToken, mockblackduckToken)
+var trustEngineConfiguration = Configuration{
+	Token:               "testToken",
+	ServerURL:           testServerURL,
+	TokenEndPoint:       testTokenEndPoint,
+	TokenQueryParamName: testTokenQueryParamName,
+}
+
+func TestTrustEngine(t *testing.T) {
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+
+	t.Run("Get Sonar token - happy path", func(t *testing.T) {
+		httpmock.RegisterResponder(http.MethodGet, testFullURL+"sonar", httpmock.NewStringResponder(200, mockSingleTokenResponse))
+
+		client := &piperhttp.Client{}
+		client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true})
+
+		token, err := GetToken("sonar", client, trustEngineConfiguration)
+		assert.NoError(t, err)
+		assert.Equal(t, mockSonarToken, token)
+	})
+
+	t.Run("Get multiple tokens - happy path", func(t *testing.T) {
+		httpmock.RegisterResponder(http.MethodGet, testFullURL+"sonar,blackduck", httpmock.NewStringResponder(200, mockTwoTokensResponse))
+
+		client := &piperhttp.Client{}
+		client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true})
+
+		secrets, err := GetSecrets([]string{"sonar", "blackduck"}, client, trustEngineConfiguration)
+
+		assert.NoError(t, err)
+		assert.Len(t, secrets, 2)
+		for _, s := range secrets {
+			switch system := s.System; system {
+			case "sonar":
+				assert.Equal(t, mockSonarToken, s.Token)
+			case "blackduck":
+				assert.Equal(t, mockblackduckToken, s.Token)
+			default:
+				continue
+			}
+		}
+	})
+
+	t.Run("Get Sonar token - 403 error", func(t *testing.T) {
+		httpmock.RegisterResponder(http.MethodGet, testFullURL+"sonar", httpmock.NewStringResponder(403, errorMsg403))
+
+		client := &piperhttp.Client{}
+		client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true})
+
+		_, err := GetToken("sonar", client, trustEngineConfiguration)
+		assert.Error(t, err)
+	})
+
+}