feat: step to execute shell scripts (#3196)

* shell executor initial commit

* functionality updates

* changes in logging implementation (using internal logging), changes in execution

* remove unused field

* remove duplicate from code

* update vault flow and remove unnecessary params

* update generated step file

Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>

cmd/metadata_generated.go

@@ -79,6 +79,7 @@ func GetAllStepMetadata() map[string]config.StepData {
 		"npmExecuteScripts":                         npmExecuteScriptsMetadata(),
 		"pipelineCreateScanSummary":                 pipelineCreateScanSummaryMetadata(),
 		"protecodeExecuteScan":                      protecodeExecuteScanMetadata(),
+		"shellExecute":                              shellExecuteMetadata(),
 		"sonarExecuteScan":                          sonarExecuteScanMetadata(),
 		"terraformExecute":                          terraformExecuteMetadata(),
 		"transportRequestDocIDFromGit":              transportRequestDocIDFromGitMetadata(),

cmd/piper.go

@@ -166,6 +166,7 @@ func Execute() {
 	rootCmd.AddCommand(InfluxWriteDataCommand())
 	rootCmd.AddCommand(AbapEnvironmentRunAUnitTestCommand())
 	rootCmd.AddCommand(CheckStepActiveCommand())
+	rootCmd.AddCommand(ShellExecuteCommand())
 	rootCmd.AddCommand(ApiProxyDownloadCommand())
 	rootCmd.AddCommand(ApiKeyValueMapDownloadCommand())
 

cmd/shellExecute.go

@@ -0,0 +1,136 @@
+package cmd
+
+import (
+	"net/url"
+	"os/exec"
+	"strings"
+
+	"github.com/hashicorp/vault/api"
+	"github.com/pkg/errors"
+
+	"github.com/SAP/jenkins-library/pkg/command"
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/piperutils"
+	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/SAP/jenkins-library/pkg/vault"
+)
+
+type shellExecuteUtils interface {
+	command.ExecRunner
+	FileExists(filename string) (bool, error)
+}
+
+type shellExecuteUtilsBundle struct {
+	*vault.Client
+	*command.Command
+	*piperutils.Files
+}
+
+func newShellExecuteUtils() shellExecuteUtils {
+	utils := shellExecuteUtilsBundle{
+		Command: &command.Command{},
+		Files:   &piperutils.Files{},
+	}
+	utils.Stdout(log.Writer())
+	utils.Stderr(log.Writer())
+	return &utils
+}
+
+func shellExecute(config shellExecuteOptions, telemetryData *telemetry.CustomData) {
+	utils := newShellExecuteUtils()
+	fileUtils := &piperutils.Files{}
+
+	err := runShellExecute(&config, telemetryData, utils, fileUtils)
+	if err != nil {
+		log.Entry().WithError(err).Fatal("step execution failed")
+	}
+}
+
+func runShellExecute(config *shellExecuteOptions, telemetryData *telemetry.CustomData, utils shellExecuteUtils, fileUtils piperutils.FileUtils) error {
+	// create vault client
+	// try to retrieve existing credentials
+	// if it's impossible - will add it
+	vaultConfig := &vault.Config{
+		Config: &api.Config{
+			Address: config.VaultServerURL,
+		},
+		Namespace: config.VaultNamespace,
+	}
+	_, err := vault.NewClientWithAppRole(vaultConfig, GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID)
+	if err != nil {
+		log.Entry().Info("could not create vault client:", err)
+	}
+
+	// piper http client for downloading scripts
+	httpClient := piperhttp.Client{}
+
+	// scripts for running locally
+	var e []string
+
+	// check input data
+	// example for script: sources: ["./script.sh"]
+	for _, source := range config.Sources {
+		// check it's a local script or remote
+		_, err := url.ParseRequestURI(source)
+		if err != nil {
+			// err means that it's not a remote script
+			// check if the script is physically present (for local scripts)
+			exists, err := fileUtils.FileExists(source)
+			if err != nil {
+				log.Entry().WithError(err).Error("failed to check for defined script")
+				return errors.Wrap(err, "failed to check for defined script")
+			}
+			if !exists {
+				log.Entry().WithError(err).Error("the specified script could not be found")
+				return errors.New("the specified script could not be found")
+			}
+			e = append(e, source)
+		} else {
+			// this block means that it's a remote script
+			// so, need to download it before
+			// get script name at first
+			path := strings.Split(source, "/")
+			err = httpClient.DownloadFile(source, path[len(path)-1], nil, nil)
+			if err != nil {
+				log.Entry().WithError(err).Errorf("the specified script could not be downloaded")
+			}
+			// make script executable
+			exec.Command("/bin/sh", "chmod +x "+path[len(path)-1])
+
+			e = append(e, path[len(path)-1])
+
+		}
+	}
+
+	// if all ok - try to run them one by one
+	for _, script := range e {
+		log.Entry().Info("starting running script:", script)
+		err = utils.RunExecutable(script)
+		if err != nil {
+			log.Entry().Errorln("starting running script:", script)
+		}
+
+		// if it's an exit error, then check the exit code
+		// according to the requirements
+		// 0 - success
+		// 1 - fails the build (or > 2)
+		// 2 - build unstable - unsupported now
+		if ee, ok := err.(*exec.ExitError); ok {
+			switch ee.ExitCode() {
+			case 0:
+				// success
+				return nil
+			case 1:
+				return errors.Wrap(err, "an error occurred while executing the script")
+			default:
+				// exit code 2 or >2 - unstable
+				return errors.Wrap(err, "script execution unstable or something went wrong")
+			}
+		} else if err != nil {
+			return errors.Wrap(err, "script execution error occurred")
+		}
+	}
+
+	return nil
+}

cmd/shellExecute_generated.go

@@ -0,0 +1,162 @@
+// Code generated by piper's step-generator. DO NOT EDIT.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/SAP/jenkins-library/pkg/config"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/splunk"
+	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/SAP/jenkins-library/pkg/validation"
+	"github.com/spf13/cobra"
+)
+
+type shellExecuteOptions struct {
+	VaultServerURL string   `json:"vaultServerUrl,omitempty"`
+	VaultNamespace string   `json:"vaultNamespace,omitempty"`
+	Sources        []string `json:"sources,omitempty"`
+}
+
+// ShellExecuteCommand Step executes defined script
+func ShellExecuteCommand() *cobra.Command {
+	const STEP_NAME = "shellExecute"
+
+	metadata := shellExecuteMetadata()
+	var stepConfig shellExecuteOptions
+	var startTime time.Time
+	var logCollector *log.CollectorHook
+	var splunkClient *splunk.Splunk
+	telemetryClient := &telemetry.Telemetry{}
+
+	var createShellExecuteCmd = &cobra.Command{
+		Use:   STEP_NAME,
+		Short: "Step executes defined script",
+		Long:  `Step executes defined script with Vault credentials, or created them on this step`,
+		PreRunE: func(cmd *cobra.Command, _ []string) error {
+			startTime = time.Now()
+			log.SetStepName(STEP_NAME)
+			log.SetVerbose(GeneralConfig.Verbose)
+
+			GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens)
+
+			path, _ := os.Getwd()
+			fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path}
+			log.RegisterHook(fatalHook)
+
+			err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile)
+			if err != nil {
+				log.SetErrorCategory(log.ErrorConfiguration)
+				return err
+			}
+
+			if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 {
+				sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID)
+				log.RegisterHook(&sentryHook)
+			}
+
+			if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+				splunkClient = &splunk.Splunk{}
+				logCollector = &log.CollectorHook{CorrelationID: GeneralConfig.CorrelationID}
+				log.RegisterHook(logCollector)
+			}
+
+			validation, err := validation.New(validation.WithJSONNamesForStructFields(), validation.WithPredefinedErrorMessages())
+			if err != nil {
+				return err
+			}
+			if err = validation.ValidateStruct(stepConfig); err != nil {
+				log.SetErrorCategory(log.ErrorConfiguration)
+				return err
+			}
+
+			return nil
+		},
+		Run: func(_ *cobra.Command, _ []string) {
+			stepTelemetryData := telemetry.CustomData{}
+			stepTelemetryData.ErrorCode = "1"
+			handler := func() {
+				config.RemoveVaultSecretFiles()
+				stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds())
+				stepTelemetryData.ErrorCategory = log.GetErrorCategory().String()
+				stepTelemetryData.PiperCommitHash = GitCommit
+				telemetryClient.SetData(&stepTelemetryData)
+				telemetryClient.Send()
+				if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+					splunkClient.Send(telemetryClient.GetData(), logCollector)
+				}
+			}
+			log.DeferExitHandler(handler)
+			defer handler()
+			telemetryClient.Initialize(GeneralConfig.NoTelemetry, STEP_NAME)
+			if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+				splunkClient.Initialize(GeneralConfig.CorrelationID,
+					GeneralConfig.HookConfig.SplunkConfig.Dsn,
+					GeneralConfig.HookConfig.SplunkConfig.Token,
+					GeneralConfig.HookConfig.SplunkConfig.Index,
+					GeneralConfig.HookConfig.SplunkConfig.SendLogs)
+			}
+			shellExecute(stepConfig, &stepTelemetryData)
+			stepTelemetryData.ErrorCode = "0"
+			log.Entry().Info("SUCCESS")
+		},
+	}
+
+	addShellExecuteFlags(createShellExecuteCmd, &stepConfig)
+	return createShellExecuteCmd
+}
+
+func addShellExecuteFlags(cmd *cobra.Command, stepConfig *shellExecuteOptions) {
+	cmd.Flags().StringVar(&stepConfig.VaultServerURL, "vaultServerUrl", os.Getenv("PIPER_vaultServerUrl"), "The URL for the Vault server to use")
+	cmd.Flags().StringVar(&stepConfig.VaultNamespace, "vaultNamespace", os.Getenv("PIPER_vaultNamespace"), "The vault namespace that should be used (optional)")
+	cmd.Flags().StringSliceVar(&stepConfig.Sources, "sources", []string{}, "Scripts names for execution or links to scripts")
+
+}
+
+// retrieve step metadata
+func shellExecuteMetadata() config.StepData {
+	var theMetaData = config.StepData{
+		Metadata: config.StepMetadata{
+			Name:        "shellExecute",
+			Aliases:     []config.Alias{},
+			Description: "Step executes defined script",
+		},
+		Spec: config.StepSpec{
+			Inputs: config.StepInputs{
+				Parameters: []config.StepParameters{
+					{
+						Name:        "vaultServerUrl",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_vaultServerUrl"),
+					},
+					{
+						Name:        "vaultNamespace",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_vaultNamespace"),
+					},
+					{
+						Name:        "sources",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "[]string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     []string{},
+					},
+				},
+			},
+		},
+	}
+	return theMetaData
+}

cmd/shellExecute_generated_test.go

@@ -0,0 +1,17 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestShellExecuteCommand(t *testing.T) {
+	t.Parallel()
+
+	testCmd := ShellExecuteCommand()
+
+	// only high level testing performed - details are tested in step generation procedure
+	assert.Equal(t, "shellExecute", testCmd.Use, "command name incorrect")
+
+}

cmd/shellExecute_test.go

@@ -0,0 +1,84 @@
+package cmd
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/SAP/jenkins-library/pkg/mock"
+)
+
+type shellExecuteMockUtils struct {
+	t      *testing.T
+	config *shellExecuteOptions
+	*mock.ExecMockRunner
+	*mock.FilesMock
+}
+
+type shellExecuteFileMock struct {
+	*mock.FilesMock
+	fileReadContent map[string]string
+	fileReadErr     map[string]error
+}
+
+func (f *shellExecuteFileMock) FileRead(path string) ([]byte, error) {
+	if f.fileReadErr[path] != nil {
+		return []byte{}, f.fileReadErr[path]
+	}
+	return []byte(f.fileReadContent[path]), nil
+}
+
+func (f *shellExecuteFileMock) FileExists(path string) (bool, error) {
+	return strings.EqualFold(path, "path/to/script/script.sh"), nil
+}
+
+func newShellExecuteTestsUtils() shellExecuteMockUtils {
+	utils := shellExecuteMockUtils{
+		ExecMockRunner: &mock.ExecMockRunner{},
+		FilesMock:      &mock.FilesMock{},
+	}
+	return utils
+}
+
+func (v *shellExecuteMockUtils) GetConfig() *shellExecuteOptions {
+	return v.config
+}
+
+func TestRunShellExecute(t *testing.T) {
+
+	t.Run("negative case - script isn't present", func(t *testing.T) {
+		c := &shellExecuteOptions{
+			Sources: []string{"path/to/script.sh"},
+		}
+		u := newShellExecuteTestsUtils()
+		fm := &shellExecuteFileMock{}
+
+		err := runShellExecute(c, nil, u, fm)
+		assert.EqualError(t, err, "the specified script could not be found")
+	})
+
+	t.Run("success case - script is present", func(t *testing.T) {
+		o := &shellExecuteOptions{}
+		u := newShellExecuteTestsUtils()
+		m := &shellExecuteFileMock{
+			fileReadContent: map[string]string{"path/to/script/script.sh": ``},
+		}
+
+		err := runShellExecute(o, nil, u, m)
+		assert.NoError(t, err)
+	})
+
+	t.Run("success case - script run successfully", func(t *testing.T) {
+		o := &shellExecuteOptions{}
+		u := newShellExecuteTestsUtils()
+		m := &shellExecuteFileMock{
+			fileReadContent: map[string]string{"path/to/script/script.sh": `#!/usr/bin/env sh
+print 'test'`},
+		}
+
+		err := runShellExecute(o, nil, u, m)
+		assert.NoError(t, err)
+	})
+
+}

pkg/config/vault.go

@@ -26,6 +26,7 @@ const (
 	vaultPath                           = "vaultPath"
 	skipVault                           = "skipVault"
 	vaultDisableOverwrite               = "vaultDisableOverwrite"
+	vaultTestCredentialEnvPrefix        = "vaultTestCredentialEnvPrefix"
 	vaultTestCredentialEnvPrefixDefault = "PIPER_TESTCREDENTIAL_"
 )
 
@@ -43,6 +44,7 @@ var (
 		vaultDisableOverwrite,
 		vaultTestCredentialPath,
 		vaultTestCredentialKeys,
+		vaultTestCredentialEnvPrefix,
 	}
 
 	// VaultRootPaths are the lookup paths piper tries to use during the vault lookup.

resources/metadata/shellExecute.yaml

@@ -0,0 +1,30 @@
+metadata:
+  name: shellExecute
+  description: Step executes defined script
+  longDescription: Step executes defined script with Vault credentials, or created them on this step
+spec:
+  inputs:
+    params:
+      - name: vaultServerUrl
+        type: string
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        description: The URL for the Vault server to use
+      - name: vaultNamespace
+        type: string
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        description: The vault namespace that should be used (optional)
+      - name: sources
+        type: "[]string"
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        description: Scripts names for execution or links to scripts