feat (shellExecute) extend sources param to download scripts. (#3674)

* first version to download script from git

* unit test adjust

* adding git token

* info messages

* removing extra info message

* changing file permission for scrtips

* modying sources to handle https download

* adding script downloads

* commenting the file permission change

* changing persmission

* adding header to download file

* adding perimssions

* adding perimssions

* not touching file permissions

* adding to pipeline

* return file name

* changing script name

* adding file permission changes

* adding file permission changes

* using current directory

* file permission

* downloading in .pipeline folder

* removing permission handeling

* improving the step docu

* improving the step docu

* unit test and code cleaning

* fix typo

* adding read execute permission

* fix unit test

* fix unit test

* removing negative test

Co-authored-by: anilkeshav27 <you@example.com>
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>

cmd/shellExecute.go

@@ -2,11 +2,15 @@ package cmd
 
 import (
 	"fmt"
+	"net/http"
 	"os/exec"
+	"path/filepath"
+	"strings"
 
 	"github.com/pkg/errors"
 
 	"github.com/SAP/jenkins-library/pkg/command"
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
@@ -15,17 +19,20 @@ import (
 type shellExecuteUtils interface {
 	command.ExecRunner
 	piperutils.FileUtils
+	piperhttp.Downloader
 }
 
 type shellExecuteUtilsBundle struct {
 	*command.Command
 	*piperutils.Files
+	*piperhttp.Client
 }
 
 func newShellExecuteUtils() shellExecuteUtils {
 	utils := shellExecuteUtilsBundle{
 		Command: &command.Command{},
 		Files:   &piperutils.Files{},
+		Client:  &piperhttp.Client{},
 	}
 	utils.Stdout(log.Writer())
 	utils.Stderr(log.Writer())
@@ -45,6 +52,14 @@ func runShellExecute(config *shellExecuteOptions, telemetryData *telemetry.Custo
 	// check input data
 	// example for script: sources: ["./script.sh"]
 	for _, source := range config.Sources {
+
+		if strings.Contains(source, "https") {
+			scriptLocation, err := downloadScript(config, utils, source)
+			if err != nil {
+				return errors.Wrapf(err, "script download error")
+			}
+			source = scriptLocation
+		}
 		// check if the script is physically present
 		exists, err := utils.FileExists(source)
 		if err != nil {
@@ -56,6 +71,7 @@ func runShellExecute(config *shellExecuteOptions, telemetryData *telemetry.Custo
 			return fmt.Errorf("the script '%v' could not be found", source)
 		}
 		log.Entry().Info("starting running script:", source)
+
 		err = utils.RunExecutable(source)
 		if err != nil {
 			log.Entry().Errorln("starting running script:", source)
@@ -79,3 +95,25 @@ func runShellExecute(config *shellExecuteOptions, telemetryData *telemetry.Custo
 
 	return nil
 }
+
+func downloadScript(config *shellExecuteOptions, utils shellExecuteUtils, url string) (string, error) {
+	header := http.Header{}
+	if len(config.GithubToken) > 0 {
+		header = http.Header{"Authorization": []string{"Token " + config.GithubToken}}
+		header.Set("Accept", "application/vnd.github.v3.raw")
+	}
+
+	log.Entry().Infof("downloading script : %v", url)
+	fileNameParts := strings.Split(url, "/")
+	fileName := fileNameParts[len(fileNameParts)-1]
+	err := utils.DownloadFile(url, filepath.Join(".pipeline", fileName), header, []*http.Cookie{})
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to download script from %v", url)
+	}
+	log.Entry().Infof("downloaded script %v successfully", url)
+	err = fileUtils.Chmod(filepath.Join(".pipeline", fileName), 0555)
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to change script permission for %v", filepath.Join(".pipeline", fileName))
+	}
+	return filepath.Join(".pipeline", fileName), nil
+}

cmd/shellExecute_generated.go

@@ -16,7 +16,8 @@ import (
 )
 
 type shellExecuteOptions struct {
-	Sources []string `json:"sources,omitempty"`
+	Sources     []string `json:"sources,omitempty"`
+	GithubToken string   `json:"githubToken,omitempty"`
 }
 
 // ShellExecuteCommand Step executes defined script
@@ -33,7 +34,7 @@ func ShellExecuteCommand() *cobra.Command {
 	var createShellExecuteCmd = &cobra.Command{
 		Use:   STEP_NAME,
 		Short: "Step executes defined script",
-		Long:  `Step executes defined script with using test Vault credentials`,
+		Long:  `Step executes defined script provided in the 'sources' parameter`,
 		PreRunE: func(cmd *cobra.Command, _ []string) error {
 			startTime = time.Now()
 			log.SetStepName(STEP_NAME)
@@ -50,6 +51,7 @@ func ShellExecuteCommand() *cobra.Command {
 				log.SetErrorCategory(log.ErrorConfiguration)
 				return err
 			}
+			log.RegisterSecret(stepConfig.GithubToken)
 
 			if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 {
 				sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID)
@@ -108,7 +110,8 @@ func ShellExecuteCommand() *cobra.Command {
 }
 
 func addShellExecuteFlags(cmd *cobra.Command, stepConfig *shellExecuteOptions) {
-	cmd.Flags().StringSliceVar(&stepConfig.Sources, "sources", []string{}, "Scripts names for execution or links to scripts")
+	cmd.Flags().StringSliceVar(&stepConfig.Sources, "sources", []string{}, "Scripts paths that must be present in the current workspace or https links to scripts. Only https urls from github are allowed and must be in the format :https://{githubBaseurl}/api/v3/repos/{owner}/{repository}/contents/{path to script} Authentication for the download is only supported via the 'githubToken' param. Make sure the script has the necessary execute permissions.")
+	cmd.Flags().StringVar(&stepConfig.GithubToken, "githubToken", os.Getenv("PIPER_githubToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line")
 
 }
 
@@ -122,6 +125,9 @@ func shellExecuteMetadata() config.StepData {
 		},
 		Spec: config.StepSpec{
 			Inputs: config.StepInputs{
+				Secrets: []config.StepSecrets{
+					{Name: "githubTokenCredentialsId", Description: "Jenkins credentials ID containing the github token.", Type: "jenkins"},
+				},
 				Parameters: []config.StepParameters{
 					{
 						Name:        "sources",
@@ -132,6 +138,26 @@ func shellExecuteMetadata() config.StepData {
 						Aliases:     []config.Alias{},
 						Default:     []string{},
 					},
+					{
+						Name: "githubToken",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name: "githubTokenCredentialsId",
+								Type: "secret",
+							},
+
+							{
+								Name:    "githubVaultSecretName",
+								Type:    "vaultSecret",
+								Default: "github",
+							},
+						},
+						Scope:     []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{{Name: "access_token"}},
+						Default:   os.Getenv("PIPER_githubToken"),
+					},
 				},
 			},
 			Containers: []config.Container{

cmd/shellExecute_test.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"net/http"
 	"strings"
 	"testing"
 
@@ -14,6 +15,11 @@ type shellExecuteMockUtils struct {
 	config *shellExecuteOptions
 	*mock.ExecMockRunner
 	*mock.FilesMock
+	*mock.HttpClientMock
+	downloadError error
+	filename      string
+	header        http.Header
+	url           string
 }
 
 type shellExecuteFileMock struct {
@@ -33,12 +39,22 @@ func (f *shellExecuteFileMock) FileExists(path string) (bool, error) {
 	return strings.EqualFold(path, "path/to/script/script.sh"), nil
 }
 
-func newShellExecuteTestsUtils() shellExecuteMockUtils {
+func (f *shellExecuteMockUtils) DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error {
+	if f.downloadError != nil {
+		return f.downloadError
+	}
+	f.url = url
+	f.filename = filename
+	f.header = header
+	return nil
+}
+
+func newShellExecuteTestsUtils() *shellExecuteMockUtils {
 	utils := shellExecuteMockUtils{
 		ExecMockRunner: &mock.ExecMockRunner{},
 		FilesMock:      &mock.FilesMock{},
 	}
-	return utils
+	return &utils
 }
 
 func (v *shellExecuteMockUtils) GetConfig() *shellExecuteOptions {
@@ -73,4 +89,15 @@ func TestRunShellExecute(t *testing.T) {
 		assert.NoError(t, err)
 	})
 
+	t.Run("success case - download script header", func(t *testing.T) {
+		o := &shellExecuteOptions{
+			Sources:     []string{"https://myScriptLocation/myScript.sh"},
+			GithubToken: "dummy@12345",
+		}
+		u := newShellExecuteTestsUtils()
+
+		runShellExecute(o, nil, u)
+
+		assert.Equal(t, http.Header{"Accept": []string{"application/vnd.github.v3.raw"}, "Authorization": []string{"Token dummy@12345"}}, u.header)
+	})
 }

resources/metadata/shellExecute.yaml

@@ -1,9 +1,13 @@
 metadata:
   name: shellExecute
   description: Step executes defined script
-  longDescription: Step executes defined script with using test Vault credentials
+  longDescription: Step executes defined script provided in the 'sources' parameter
 spec:
   inputs:
+    secrets:
+      - name: githubTokenCredentialsId
+        description: Jenkins credentials ID containing the github token.
+        type: jenkins
     params:
       - name: sources
         type: "[]string"
@@ -11,7 +15,27 @@ spec:
           - PARAMETERS
           - STAGES
           - STEPS
-        description: Scripts names for execution or links to scripts
+        description: Scripts paths that must be present in the current workspace or https links to scripts.
+            Only https urls from github are allowed and must be in the format :https://{githubBaseurl}/api/v3/repos/{owner}/{repository}/contents/{path to script}
+            Authentication for the download is only supported via the 'githubToken' param. Make sure the script has the necessary execute permissions.
+      - name: githubToken
+        description: "GitHub personal access token as per
+          https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line"
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: string
+        secret: true
+        aliases:
+          - name: access_token
+        resourceRef:
+          - name: githubTokenCredentialsId
+            type: secret
+          - type: vaultSecret
+            default: github
+            name: githubVaultSecretName
   containers:
     - name: shell
       image: node:lts-stretch

vars/shellExecute.groovy

@@ -4,6 +4,6 @@ import groovy.transform.Field
 @Field String METADATA_FILE = 'metadata/shellExecute.yaml'
 
 void call(Map parameters = [:]) {
-    List credentials = []
+     List credentials = [[type: 'token', id: 'githubTokenCredentialsId', env: ['PIPER_githubToken']]]
     piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials)
 }