Mta extension credentials handling (#2430)

Mta extension credentials handling

Co-authored-by: Stephan Aßmus <stephan.assmus@sap.com>

cmd/cloudFoundryDeploy.go

@@ -15,17 +15,23 @@ import (
 	"io"
 	"os"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
+	"unicode"
 )
 
 type cfFileUtil interface {
 	FileExists(string) (bool, error)
+	FileRename(string, string) error
+	FileRead(string) ([]byte, error)
 	FileWrite(path string, content []byte, perm os.FileMode) error
 	Getwd() (string, error)
 	Glob(string) ([]string, error)
 	Chmod(string, os.FileMode) error
+	Copy(string, string) (int64, error)
+	Stat(path string) (os.FileInfo, error)
 }
 
 var _now = time.Now
@@ -35,6 +41,7 @@ var _getManifest = getManifest
 var _replaceVariables = yaml.Substitute
 var _getVarsOptions = cloudfoundry.GetVarsOptions
 var _getVarsFileOptions = cloudfoundry.GetVarsFileOptions
+var _environ = os.Environ
 var fileUtils cfFileUtil = piperutils.Files{}
 
 // for simplify mocking. Maybe we find a more elegant way (mock for CFUtils)
@@ -668,21 +675,141 @@ func deployMta(config *cloudFoundryDeployOptions, mtarFilePath string, command c
 		cfDeployParams = append(cfDeployParams, deployParams...)
 	}
 
-	cfDeployParams = append(cfDeployParams, handleMtaExtensionDescriptors(config.MtaExtensionDescriptor)...)
+	extFileParams, extFiles := handleMtaExtensionDescriptors(config.MtaExtensionDescriptor)
 
-	return cfDeploy(config, cfDeployParams, nil, nil, command)
+	for _, extFile := range extFiles {
+		_, err := fileUtils.Copy(extFile, extFile+".original")
+		if err != nil {
+			return fmt.Errorf("Cannot prepare mta extension files: %w", err)
+		}
+		err = handleMtaExtensionCredentials(extFile, config.MtaExtensionCredentials)
+		if err != nil {
+			return fmt.Errorf("Cannot handle credentials inside mta extension files: %w", err)
+		}
+	}
+
+	cfDeployParams = append(cfDeployParams, extFileParams...)
+
+	err := cfDeploy(config, cfDeployParams, nil, nil, command)
+
+	for _, extFile := range extFiles {
+		renameError := fileUtils.FileRename(extFile+".original", extFile)
+		if err == nil && renameError != nil {
+			return renameError
+		}
+	}
+
+	return err
 }
 
-func handleMtaExtensionDescriptors(mtaExtensionDescriptor string) []string {
+func handleMtaExtensionCredentials(extFile string, credentials map[string]interface{}) error {
+
+	log.Entry().Debugf("Inserting credentials into extension file '%s'", extFile)
+
+	b, err := fileUtils.FileRead(extFile)
+	if err != nil {
+		return errors.Wrapf(err, "Cannot handle credentials for mta extension file '%s'", extFile)
+	}
+	content := string(b)
+
+	env, err := toMap(_environ(), "=")
+	if err != nil {
+		errors.Wrap(err, "Cannot handle mta extension credentials.")
+	}
+
+	updated := false
+	missingCredentials := []string{}
+	for name, credentialKey := range credentials {
+		credKey, ok := credentialKey.(string)
+		if !ok {
+			return fmt.Errorf("Cannot handle mta extension credentials: Cannot cast '%v' (type %T) to string", credentialKey, credentialKey)
+		}
+		pattern := "<%= " + name + " %>"
+		if strings.Contains(content, pattern) {
+			cred := env[toEnvVarKey(credKey)]
+			if len(cred) == 0 {
+				missingCredentials = append(missingCredentials, credKey)
+				continue
+			}
+			content = strings.Replace(content, pattern, cred, -1)
+			updated = true
+			log.Entry().Debugf("Mta extension credentials handling: Placeholder '%s' has been replaced by credential denoted by '%s'/'%s' in file '%s'", name, credKey, toEnvVarKey(credKey), extFile)
+		}
+	}
+	if len(missingCredentials) > 0 {
+		missinCredsEnvVarKeyCompatible := []string{}
+		for _, missingKey := range missingCredentials {
+			missinCredsEnvVarKeyCompatible = append(missinCredsEnvVarKeyCompatible, toEnvVarKey(missingKey))
+		}
+		// ensure stable order of the entries. Needed e.g. for the tests.
+		sort.Strings(missingCredentials)
+		sort.Strings(missinCredsEnvVarKeyCompatible)
+		return fmt.Errorf("Cannot handle mta extension credentials: No credentials found for '%s'/'%s'. Are these credentials maintained?", missingCredentials, missinCredsEnvVarKeyCompatible)
+	}
+	if !updated {
+		log.Entry().Debugf("Mta extension credentials handling: Extension file '%s' has not been updated. Seems to contain no credentials.", extFile)
+	} else {
+		fInfo, err := fileUtils.Stat(extFile)
+		fMode := fInfo.Mode()
+		if err != nil {
+			errors.Wrap(err, "Cannot handle mta extension credentials.")
+		}
+		err = fileUtils.FileWrite(extFile, []byte(content), fMode)
+		if err != nil {
+			return errors.Wrap(err, "Cannot handle mta extension credentials.")
+		}
+		log.Entry().Debugf("Mta extension credentials handling: Extension file '%s' has been updated.", extFile)
+	}
+
+	re := regexp.MustCompile(`<%= .* %>`)
+	placeholders := re.FindAll([]byte(content), -1)
+	if len(placeholders) > 0 {
+		log.Entry().Warningf("mta extension credential handling: Unresolved placeholders found after inserting credentials: %s", placeholders)
+	}
+
+	return nil
+}
+
+func toEnvVarKey(key string) string {
+	key = regexp.MustCompile(`[^A-Za-z0-9]`).ReplaceAllString(key, "_")
+	// from here on we have only ascii
+	modifiedKey := ""
+	last := '_'
+	for _, runeVal := range key {
+		if unicode.IsUpper(runeVal) && last != '_' {
+			modifiedKey += "_"
+		}
+		modifiedKey += string(unicode.ToUpper(runeVal))
+		last = runeVal
+	}
+	return modifiedKey
+	// since golang regex does not support negative lookbehinds we have to code it ourselvs
+}
+
+func toMap(keyValue []string, separator string) (map[string]string, error) {
+	result := map[string]string{}
+	for _, entry := range keyValue {
+		kv := strings.Split(entry, separator)
+		if len(kv) < 2 {
+			return map[string]string{}, fmt.Errorf("Cannot convert to map: separator '%s' not found in entry '%s'", separator, entry)
+		}
+		result[kv[0]] = strings.Join(kv[1:], separator)
+	}
+	return result, nil
+}
+
+func handleMtaExtensionDescriptors(mtaExtensionDescriptor string) ([]string, []string) {
 	var result = []string{}
+	var extFiles = []string{}
 	for _, part := range strings.Fields(strings.Trim(mtaExtensionDescriptor, " ")) {
 		if part == "-e" || part == "" {
 			continue
 		}
 		// REVISIT: maybe check if the extension descriptor exists
 		result = append(result, "-e", part)
+		extFiles = append(extFiles, part)
 	}
-	return result
+	return result, extFiles
 }
 
 func cfDeploy(

cmd/cloudFoundryDeploy_generated.go

@@ -16,32 +16,33 @@ import (
 )
 
 type cloudFoundryDeployOptions struct {
-	APIEndpoint              string   `json:"apiEndpoint,omitempty"`
+	APIEndpoint              string                 `json:"apiEndpoint,omitempty"`
-	AppName                  string   `json:"appName,omitempty"`
+	AppName                  string                 `json:"appName,omitempty"`
-	ArtifactVersion          string   `json:"artifactVersion,omitempty"`
+	ArtifactVersion          string                 `json:"artifactVersion,omitempty"`
-	CfHome                   string   `json:"cfHome,omitempty"`
+	CfHome                   string                 `json:"cfHome,omitempty"`
-	CfNativeDeployParameters string   `json:"cfNativeDeployParameters,omitempty"`
+	CfNativeDeployParameters string                 `json:"cfNativeDeployParameters,omitempty"`
-	CfPluginHome             string   `json:"cfPluginHome,omitempty"`
+	CfPluginHome             string                 `json:"cfPluginHome,omitempty"`
-	DeployDockerImage        string   `json:"deployDockerImage,omitempty"`
+	DeployDockerImage        string                 `json:"deployDockerImage,omitempty"`
-	DeployTool               string   `json:"deployTool,omitempty"`
+	DeployTool               string                 `json:"deployTool,omitempty"`
-	BuildTool                string   `json:"buildTool,omitempty"`
+	BuildTool                string                 `json:"buildTool,omitempty"`
-	DeployType               string   `json:"deployType,omitempty"`
+	DeployType               string                 `json:"deployType,omitempty"`
-	DockerPassword           string   `json:"dockerPassword,omitempty"`
+	DockerPassword           string                 `json:"dockerPassword,omitempty"`
-	DockerUsername           string   `json:"dockerUsername,omitempty"`
+	DockerUsername           string                 `json:"dockerUsername,omitempty"`
-	KeepOldInstance          bool     `json:"keepOldInstance,omitempty"`
+	KeepOldInstance          bool                   `json:"keepOldInstance,omitempty"`
-	LoginParameters          string   `json:"loginParameters,omitempty"`
+	LoginParameters          string                 `json:"loginParameters,omitempty"`
-	Manifest                 string   `json:"manifest,omitempty"`
+	Manifest                 string                 `json:"manifest,omitempty"`
-	ManifestVariables        []string `json:"manifestVariables,omitempty"`
+	ManifestVariables        []string               `json:"manifestVariables,omitempty"`
-	ManifestVariablesFiles   []string `json:"manifestVariablesFiles,omitempty"`
+	ManifestVariablesFiles   []string               `json:"manifestVariablesFiles,omitempty"`
-	MtaDeployParameters      string   `json:"mtaDeployParameters,omitempty"`
+	MtaDeployParameters      string                 `json:"mtaDeployParameters,omitempty"`
-	MtaExtensionDescriptor   string   `json:"mtaExtensionDescriptor,omitempty"`
+	MtaExtensionDescriptor   string                 `json:"mtaExtensionDescriptor,omitempty"`
-	MtaPath                  string   `json:"mtaPath,omitempty"`
+	MtaExtensionCredentials  map[string]interface{} `json:"mtaExtensionCredentials,omitempty"`
-	Org                      string   `json:"org,omitempty"`
+	MtaPath                  string                 `json:"mtaPath,omitempty"`
-	Password                 string   `json:"password,omitempty"`
+	Org                      string                 `json:"org,omitempty"`
-	SmokeTestScript          string   `json:"smokeTestScript,omitempty"`
+	Password                 string                 `json:"password,omitempty"`
-	SmokeTestStatusCode      int      `json:"smokeTestStatusCode,omitempty"`
+	SmokeTestScript          string                 `json:"smokeTestScript,omitempty"`
-	Space                    string   `json:"space,omitempty"`
+	SmokeTestStatusCode      int                    `json:"smokeTestStatusCode,omitempty"`
-	Username                 string   `json:"username,omitempty"`
+	Space                    string                 `json:"space,omitempty"`
+	Username                 string                 `json:"username,omitempty"`
 }
 
 type cloudFoundryDeployInflux struct {
@@ -175,6 +176,7 @@ func addCloudFoundryDeployFlags(cmd *cobra.Command, stepConfig *cloudFoundryDepl
 	cmd.Flags().StringSliceVar(&stepConfig.ManifestVariablesFiles, "manifestVariablesFiles", []string{`manifest-variables.yml`}, "path(s) of the Yaml file(s) containing the variable values to use as a replacement in the manifest file. The order of the files is relevant in case there are conflicting variable names and values within variable files. In such a case, the values of the last file win.")
 	cmd.Flags().StringVar(&stepConfig.MtaDeployParameters, "mtaDeployParameters", `-f`, "Additional parameters passed to mta deployment command")
 	cmd.Flags().StringVar(&stepConfig.MtaExtensionDescriptor, "mtaExtensionDescriptor", os.Getenv("PIPER_mtaExtensionDescriptor"), "Defines additional extension descriptor file for deployment with the mtaDeployPlugin")
+
 	cmd.Flags().StringVar(&stepConfig.MtaPath, "mtaPath", os.Getenv("PIPER_mtaPath"), "Defines the path to *.mtar for deployment with the mtaDeployPlugin")
 	cmd.Flags().StringVar(&stepConfig.Org, "org", os.Getenv("PIPER_org"), "Cloud Foundry target organization.")
 	cmd.Flags().StringVar(&stepConfig.Password, "password", os.Getenv("PIPER_password"), "Password")
@@ -375,6 +377,14 @@ func cloudFoundryDeployMetadata() config.StepData {
 						Mandatory:   false,
 						Aliases:     []config.Alias{{Name: "cloudFoundry/mtaExtensionDescriptor"}},
 					},
+					{
+						Name:        "mtaExtensionCredentials",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS", "GENERAL"},
+						Type:        "map[string]interface{}",
+						Mandatory:   false,
+						Aliases:     []config.Alias{{Name: "cloudFoundry/mtaExtensionCredentials"}},
+					},
 					{
 						Name: "mtaPath",
 						ResourceRef: []config.ResourceReference{

cmd/cloudFoundryDeploy_test.go

@@ -1216,7 +1216,7 @@ func TestDefaultManifestVariableFilesHandling(t *testing.T) {
 func TestExtensionDescriptorsWithMinusE(t *testing.T) {
 
 	t.Run("ExtensionDescriptorsWithMinusE", func(t *testing.T) {
-		extDesc := handleMtaExtensionDescriptors("-e 1.yaml -e 2.yaml")
+		extDesc, _ := handleMtaExtensionDescriptors("-e 1.yaml -e 2.yaml")
 		assert.Equal(t, []string{
 			"-e",
 			"1.yaml",
@@ -1226,7 +1226,7 @@ func TestExtensionDescriptorsWithMinusE(t *testing.T) {
 	})
 
 	t.Run("ExtensionDescriptorsFirstOneWithoutMinusE", func(t *testing.T) {
-		extDesc := handleMtaExtensionDescriptors("1.yaml -e 2.yaml")
+		extDesc, _ := handleMtaExtensionDescriptors("1.yaml -e 2.yaml")
 		assert.Equal(t, []string{
 			"-e",
 			"1.yaml",
@@ -1236,7 +1236,7 @@ func TestExtensionDescriptorsWithMinusE(t *testing.T) {
 	})
 
 	t.Run("NoExtensionDescriptors", func(t *testing.T) {
-		extDesc := handleMtaExtensionDescriptors("")
+		extDesc, _ := handleMtaExtensionDescriptors("")
 		assert.Equal(t, []string{}, extDesc)
 	})
 }
@@ -1277,3 +1277,88 @@ func TestAppNameChecks(t *testing.T) {
 	})
 
 }
+
+func TestMtaExtensionCredentials(t *testing.T) {
+
+	content := []byte(`'_schema-version: '3.1'
+	ID: test.ext
+	extends: test
+	parameters
+		test-credentials1: "<%= testCred1 %>"
+		test-credentials2: "<%= testCred2 %>"`)
+
+	filesMock := mock.FilesMock{}
+	filesMock.AddDir("/home/me")
+	filesMock.Chdir("/home/me")
+	filesMock.AddFile("mtaext1.mtaext", content)
+	filesMock.AddFile("mtaext2.mtaext", content)
+	filesMock.AddFile("mtaext3.mtaext", content)
+	fileUtils = &filesMock
+
+	_environ = func() []string {
+		return []string{
+			"MY_CRED_ENV_VAR1=******",
+			"MY_CRED_ENV_VAR2=++++++",
+		}
+	}
+
+	defer func() {
+		fileUtils = piperutils.Files{}
+		_environ = os.Environ
+	}()
+
+	t.Run("extension file does not exist", func(t *testing.T) {
+		err := handleMtaExtensionCredentials("mtaextDoesNotExist.mtaext", map[string]interface{}{})
+		assert.EqualError(t, err, "Cannot handle credentials for mta extension file 'mtaextDoesNotExist.mtaext': could not read 'mtaextDoesNotExist.mtaext'")
+	})
+
+	t.Run("credential cannot be retrieved", func(t *testing.T) {
+
+		err := handleMtaExtensionCredentials(
+			"mtaext1.mtaext",
+			map[string]interface{}{
+				"testCred1": "myCredEnvVar1NotDefined",
+				"testCred2": "myCredEnvVar2NotDefined",
+			},
+		)
+		assert.EqualError(t, err, "Cannot handle mta extension credentials: No credentials found for '[myCredEnvVar1NotDefined myCredEnvVar2NotDefined]'/'[MY_CRED_ENV_VAR1_NOT_DEFINED MY_CRED_ENV_VAR2_NOT_DEFINED]'. Are these credentials maintained?")
+	})
+
+	t.Run("irrelevant credentials does not cause failures", func(t *testing.T) {
+
+		err := handleMtaExtensionCredentials(
+			"mtaext2.mtaext",
+			map[string]interface{}{
+				"testCred1":       "myCredEnvVar1",
+				"testCred2":       "myCredEnvVar2",
+				"testCredNotUsed": "myCredEnvVarWhichDoesNotExist", //<-- This here is not used.
+			},
+		)
+		assert.NoError(t, err)
+	})
+
+	t.Run("replace straight forward", func(t *testing.T) {
+		mtaFileName := "mtaext3.mtaext"
+		err := handleMtaExtensionCredentials(
+			mtaFileName,
+			map[string]interface{}{
+				"testCred1": "myCredEnvVar1",
+				"testCred2": "myCredEnvVar2",
+			},
+		)
+		if assert.NoError(t, err) {
+			b, e := fileUtils.FileRead(mtaFileName)
+			if e != nil {
+				assert.Fail(t, "Cannot read mta extension file: %v", e)
+			}
+			content := string(b)
+			assert.Contains(t, content, "test-credentials1: \"******\"")
+			assert.Contains(t, content, "test-credentials2: \"++++++\"")
+		}
+	})
+}
+
+func TestEnvVarKeyModification(t *testing.T) {
+	envVarCompatibleKey := toEnvVarKey("Mta.ExtensionCredential~Credential_Id1")
+	assert.Equal(t, "MTA_EXTENSION_CREDENTIAL_CREDENTIAL_ID1", envVarCompatibleKey)
+}

resources/metadata/cloudFoundryDeploy.yaml

@@ -259,6 +259,17 @@ spec:
         mandatory: false
         aliases:
           - name: cloudFoundry/mtaExtensionDescriptor
+      - name: mtaExtensionCredentials
+        type: "map[string]interface{}"
+        description: "Defines a map of credentials that need to be replaced in the `mtaExtensionDescriptor`. This map needs to be created as `value-to-be-replaced`:`id-of-a-credential-in-jenkins`"
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+          - GENERAL
+        mandatory: false
+        aliases:
+          - name: cloudFoundry/mtaExtensionCredentials
       - name: mtaPath
         type: string
         description: "Defines the path to *.mtar for deployment with the mtaDeployPlugin"