Introducing new step 'gctsCloneRepository' (#1468)

* added new step gctsCloneRepository

* enhanced docu

* adjusted test case name

* enhanced docu

* refined error messages

* regenerated & remove unused Groovy imports

* small change

* remove execRunner

* Update documentation/docs/steps/gctsCloneRepository.md

* regenerated

Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>

cmd/gctsCloneRepository.go

@@ -0,0 +1,95 @@
+package cmd
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/cookiejar"
+
+	"github.com/Jeffail/gabs/v2"
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/pkg/errors"
+)
+
+func gctsCloneRepository(config gctsCloneRepositoryOptions, telemetryData *telemetry.CustomData) {
+
+	// for http calls import  piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	// and use a  &piperhttp.Client{} in a custom system
+	// Example: step checkmarxExecuteScan.go
+	httpClient := &piperhttp.Client{}
+
+	// error situations should stop execution through log.Entry().Fatal() call which leads to an os.Exit(1) in the end
+	err := cloneRepository(&config, telemetryData, httpClient)
+	if err != nil {
+		log.Entry().WithError(err).Fatal("step execution failed")
+	}
+}
+
+func cloneRepository(config *gctsCloneRepositoryOptions, telemetryData *telemetry.CustomData, httpClient piperhttp.Sender) error {
+
+	cookieJar, cookieErr := cookiejar.New(nil)
+	if cookieErr != nil {
+		return errors.Wrap(cookieErr, "creating a cookie jar failed")
+	}
+	clientOptions := piperhttp.ClientOptions{
+		CookieJar: cookieJar,
+		Username:  config.Username,
+		Password:  config.Password,
+	}
+	httpClient.SetOptions(clientOptions)
+
+	header := make(http.Header)
+	header.Set("Content-Type", "application/json")
+	header.Add("Accept", "application/json")
+
+	url := config.Host +
+		"/sap/bc/cts_abapvcs/repository/" + config.Repository +
+		"/clone?sap-client=" + config.Client
+
+	resp, httpErr := httpClient.SendRequest("POST", url, nil, header, nil)
+
+	defer func() {
+		if resp != nil && resp.Body != nil {
+			resp.Body.Close()
+		}
+	}()
+
+	if resp == nil {
+		return errors.Errorf("did not retrieve a HTTP response: %v", httpErr)
+	}
+
+	bodyText, readErr := ioutil.ReadAll(resp.Body)
+
+	if readErr != nil {
+		return errors.Wrap(readErr, "HTTP response body could not be read")
+	}
+
+	response, parsingErr := gabs.ParseJSON([]byte(bodyText))
+
+	if parsingErr != nil {
+		return errors.Wrapf(parsingErr, "HTTP response body could not be parsed as JSON: %v", string(bodyText))
+	}
+
+	if httpErr != nil {
+		if resp.StatusCode == 500 {
+			if exception, ok := response.Path("errorLog.1.code").Data().(string); ok && exception == "GCTS.CLIENT.1420" {
+				log.Entry().
+					WithField("repository", config.Repository).
+					Info("the repository has already been cloned")
+				return nil
+			} else if exception, ok := response.Path("errorLog.1.code").Data().(string); ok && exception == "GCTS.CLIENT.3302" {
+				log.Entry().Errorf("%v", response.Path("errorLog.1.message").Data().(string))
+				log.Entry().Error("possible reason: the remote repository is set to 'private'. you need to provide the local ABAP server repository with authentication credentials to the remote Git repository in order to clone it.")
+				return errors.Wrap(httpErr, "cloning the repository failed")
+			}
+		}
+		log.Entry().Errorf("a HTTP error occured! Response body: %v", response)
+		return errors.Wrap(httpErr, "cloning the repository failed")
+	}
+
+	log.Entry().
+		WithField("repository", config.Repository).
+		Info("successfully cloned the Git repository to the local repository")
+	return nil
+}

cmd/gctsCloneRepository_generated.go

@@ -0,0 +1,148 @@
+// Code generated by piper's step-generator. DO NOT EDIT.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/SAP/jenkins-library/pkg/config"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/spf13/cobra"
+)
+
+type gctsCloneRepositoryOptions struct {
+	Username   string `json:"username,omitempty"`
+	Password   string `json:"password,omitempty"`
+	Repository string `json:"repository,omitempty"`
+	Host       string `json:"host,omitempty"`
+	Client     string `json:"client,omitempty"`
+}
+
+// GctsCloneRepositoryCommand Clones a Git repository
+func GctsCloneRepositoryCommand() *cobra.Command {
+	const STEP_NAME = "gctsCloneRepository"
+
+	metadata := gctsCloneRepositoryMetadata()
+	var stepConfig gctsCloneRepositoryOptions
+	var startTime time.Time
+
+	var createGctsCloneRepositoryCmd = &cobra.Command{
+		Use:   STEP_NAME,
+		Short: "Clones a Git repository",
+		Long:  `Clones a Git repository from a remote repository to a local repository on an ABAP system. To be able to execute this step, the corresponding local repository has to exist on the local ABAP system.`,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			startTime = time.Now()
+			log.SetStepName(STEP_NAME)
+			log.SetVerbose(GeneralConfig.Verbose)
+
+			path, _ := os.Getwd()
+			fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path}
+			log.RegisterHook(fatalHook)
+
+			err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile)
+			if err != nil {
+				return err
+			}
+			log.RegisterSecret(stepConfig.Username)
+			log.RegisterSecret(stepConfig.Password)
+
+			if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 {
+				sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID)
+				log.RegisterHook(&sentryHook)
+			}
+
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			telemetryData := telemetry.CustomData{}
+			telemetryData.ErrorCode = "1"
+			handler := func() {
+				telemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds())
+				telemetry.Send(&telemetryData)
+			}
+			log.DeferExitHandler(handler)
+			defer handler()
+			telemetry.Initialize(GeneralConfig.NoTelemetry, STEP_NAME)
+			gctsCloneRepository(stepConfig, &telemetryData)
+			telemetryData.ErrorCode = "0"
+			log.Entry().Info("SUCCESS")
+		},
+	}
+
+	addGctsCloneRepositoryFlags(createGctsCloneRepositoryCmd, &stepConfig)
+	return createGctsCloneRepositoryCmd
+}
+
+func addGctsCloneRepositoryFlags(cmd *cobra.Command, stepConfig *gctsCloneRepositoryOptions) {
+	cmd.Flags().StringVar(&stepConfig.Username, "username", os.Getenv("PIPER_username"), "User to authenticate to the ABAP system")
+	cmd.Flags().StringVar(&stepConfig.Password, "password", os.Getenv("PIPER_password"), "Password to authenticate to the ABAP system")
+	cmd.Flags().StringVar(&stepConfig.Repository, "repository", os.Getenv("PIPER_repository"), "Specifies the name (ID) of the local repsitory on the ABAP system")
+	cmd.Flags().StringVar(&stepConfig.Host, "host", os.Getenv("PIPER_host"), "Specifies the protocol and host adress, including the port. Please provide in the format '<protocol>://<host>:<port>'")
+	cmd.Flags().StringVar(&stepConfig.Client, "client", os.Getenv("PIPER_client"), "Specifies the client of the ABAP system to be adressed")
+
+	cmd.MarkFlagRequired("username")
+	cmd.MarkFlagRequired("password")
+	cmd.MarkFlagRequired("repository")
+	cmd.MarkFlagRequired("host")
+	cmd.MarkFlagRequired("client")
+}
+
+// retrieve step metadata
+func gctsCloneRepositoryMetadata() config.StepData {
+	var theMetaData = config.StepData{
+		Metadata: config.StepMetadata{
+			Name:    "gctsCloneRepository",
+			Aliases: []config.Alias{},
+		},
+		Spec: config.StepSpec{
+			Inputs: config.StepInputs{
+				Parameters: []config.StepParameters{
+					{
+						Name:        "username",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   true,
+						Aliases:     []config.Alias{},
+					},
+					{
+						Name:        "password",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   true,
+						Aliases:     []config.Alias{},
+					},
+					{
+						Name:        "repository",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   true,
+						Aliases:     []config.Alias{},
+					},
+					{
+						Name:        "host",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   true,
+						Aliases:     []config.Alias{},
+					},
+					{
+						Name:        "client",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   true,
+						Aliases:     []config.Alias{},
+					},
+				},
+			},
+		},
+	}
+	return theMetaData
+}

cmd/gctsCloneRepository_generated_test.go

@@ -0,0 +1,16 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGctsCloneRepositoryCommand(t *testing.T) {
+
+	testCmd := GctsCloneRepositoryCommand()
+
+	// only high level testing performed - details are tested in step generation procudure
+	assert.Equal(t, "gctsCloneRepository", testCmd.Use, "command name incorrect")
+
+}

cmd/gctsCloneRepository_test.go

@@ -0,0 +1,207 @@
+package cmd
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestGctsCloneRepositorySuccess(t *testing.T) {
+
+	config := gctsCloneRepositoryOptions{
+		Host:       "http://testHost.com:50000",
+		Client:     "000",
+		Repository: "testRepo",
+		Username:   "testUser",
+		Password:   "testPassword",
+	}
+
+	t.Run("cloning successfull", func(t *testing.T) {
+
+		httpClient := httpMockGcts{StatusCode: 200, ResponseBody: `{
+			"result": {
+				"rid": "my-repository",
+				"checkoutTime": 20180606130524,
+				"fromCommit": "f1cdb6a032c1d8187c0990b51e94e8d8bb9898b2",
+				"toCommit": "f1cdb6a032c1d8187c0990b51e94e8d8bb9898b2",
+				"caller": "JOHNDOE",
+				"request": "SIDK1234567",
+				"type": "PULL"
+			},
+			"log": [
+				{
+					"time": 20180606130524,
+					"user": "JENKINS",
+					"section": "REPOSITORY_FACTORY",
+					"action": "CREATE_REPOSITORY",
+					"severity": "INFO",
+					"message": "Start action CREATE_REPOSITORY review",
+					"code": "GCTS.API.410"
+				}
+			]
+		}`}
+
+		err := cloneRepository(&config, nil, &httpClient)
+
+		if assert.NoError(t, err) {
+
+			t.Run("check url", func(t *testing.T) {
+				assert.Equal(t, "http://testHost.com:50000/sap/bc/cts_abapvcs/repository/testRepo/clone?sap-client=000", httpClient.URL)
+			})
+
+			t.Run("check method", func(t *testing.T) {
+				assert.Equal(t, "POST", httpClient.Method)
+			})
+
+			t.Run("check user", func(t *testing.T) {
+				assert.Equal(t, "testUser", httpClient.Options.Username)
+			})
+
+			t.Run("check password", func(t *testing.T) {
+				assert.Equal(t, "testPassword", httpClient.Options.Password)
+			})
+
+		}
+
+	})
+
+	t.Run("repository has already been cloned", func(t *testing.T) {
+
+		httpClient := httpMockGcts{StatusCode: 500, ResponseBody: `{
+			"errorLog": [
+				{
+					"time": 20200414112900,
+					"user": "USER",
+					"section": "CLIENT_TP",
+					"action": "GET_SOURCE_FROM_REMOTE",
+					"severity": "ERROR",
+					"message": "20200414112900: Error action GET_SOURCE_FROM_REMOTE",
+					"protocol": [
+						{
+							"type": "Paramters",
+							"protocol": [
+								{
+									"repouri": "https://github.com/test-repo",
+									"logfile": "/usr/sap/SID/D00/gcts/repos/gcts/repo-name/log/20200414_112900_AD0F43952A5A3F47133637329BA760D8.log",
+									"repodir": "/usr/sap/SID/D00/gcts/repos/gcts/repo-name/data/",
+									"loglevel": "warning",
+									"command": "clone"
+								}
+							]
+						},
+						{
+							"type": "Client Log",
+							"protocol": [
+								"protocol logs"
+							]
+						},
+						{
+							"type": "Client Stack Log",
+							"protocol": [
+								{
+									"client stack log key": "client stack log value",
+									"stack": [
+										"java",
+										"stack"
+									]
+								}
+							]
+						}
+					]
+				},
+				{
+					"severity": "ERROR",
+					"message": "Cloning a repository into a new working directory failed: Destination path 'data' already exists and is not an empty directory",
+					"code": "GCTS.CLIENT.1420"
+				},
+				{
+					"time": 20200414112900,
+					"user": "USER",
+					"section": "REPOSITORY",
+					"action": "CLONE",
+					"severity": "ERROR",
+					"message": "20200414112900: Error action CLONE 20200414_112900_AD0F43952A5A3F47133637329BA760D8"
+				}
+			]
+		}`}
+
+		err := cloneRepository(&config, nil, &httpClient)
+		assert.NoError(t, err)
+
+	})
+
+}
+
+func TestGctsCloneRepositoryFailure(t *testing.T) {
+
+	config := gctsCloneRepositoryOptions{
+		Host:       "http://testHost.com:50000",
+		Client:     "000",
+		Repository: "testRepo",
+		Username:   "testUser",
+		Password:   "testPassword",
+	}
+
+	t.Run("a http error occurred", func(t *testing.T) {
+		httpClient := httpMockGcts{StatusCode: 500, ResponseBody: `{
+			"errorLog": [
+				{
+					"time": 20200414112900,
+					"user": "USER",
+					"section": "CLIENT_TP",
+					"action": "GET_SOURCE_FROM_REMOTE",
+					"severity": "ERROR",
+					"message": "20200414112900: Error action GET_SOURCE_FROM_REMOTE",
+					"protocol": [
+						{
+							"type": "Paramters",
+							"protocol": [
+								{
+									"repouri": "https://github.com/test-repo",
+									"logfile": "/usr/sap/SID/D00/gcts/repos/gcts/repo-name/log/20200414_112900_AD0F43952A5A3F47133637329BA760D8.log",
+									"repodir": "/usr/sap/SID/D00/gcts/repos/gcts/repo-name/data/",
+									"loglevel": "warning",
+									"command": "clone"
+								}
+							]
+						},
+						{
+							"type": "Client Log",
+							"protocol": [
+								"protocol logs"
+							]
+						},
+						{
+							"type": "Client Stack Log",
+							"protocol": [
+								{
+									"client stack log key": "client stack log value",
+									"stack": [
+										"java",
+										"stack"
+									]
+								}
+							]
+						}
+					]
+				},
+				{
+					"severity": "ERROR",
+					"message": "Cloning a repository into a new working directory failed: Destination path 'data' already exists and is not an empty directory",
+					"code": "GCTS.CLIENT.9999"
+				},
+				{
+					"time": 20200414112900,
+					"user": "USER",
+					"section": "REPOSITORY",
+					"action": "CLONE",
+					"severity": "ERROR",
+					"message": "20200414112900: Error action CLONE 20200414_112900_AD0F43952A5A3F47133637329BA760D8"
+				}
+			]
+		}`}
+
+		err := cloneRepository(&config, nil, &httpClient)
+		assert.EqualError(t, err, "cloning the repository failed: a http error occurred")
+
+	})
+}

cmd/piper.go

@@ -87,6 +87,7 @@ func Execute() {
 	rootCmd.AddCommand(GctsCreateRepositoryCommand())
 	rootCmd.AddCommand(GctsDeployCommand())
 	rootCmd.AddCommand(MalwareExecuteScanCommand())
+	rootCmd.AddCommand(GctsCloneRepositoryCommand())
 
 	addRootFlags(rootCmd)
 	if err := rootCmd.Execute(); err != nil {

documentation/docs/steps/gctsCloneRepository.md

@@ -0,0 +1,41 @@
+# ${docGenStepName}
+
+## ${docGenDescription}
+
+## Prerequisites
+
+With this step you can clone a remote Git repository to a local repository on an ABAP server. To be able to execute this step, the corresponding local repository has to exist on the local ABAP system.
+Learn more about the SAP git-enabled Central Transport Sytem (gCTS) [here](https://help.sap.com/viewer/4a368c163b08418890a406d413933ba7/201909.001/en-US/f319b168e87e42149e25e13c08d002b9.html). With gCTS, ABAP developments on ABAP servers can be maintained in Git repositories.
+
+## ${docGenParameters}
+
+## ${docGenConfiguration}
+
+## ${docJenkinsPluginDependencies}
+
+## Example
+
+Example configuration for the use in a `Jenkinsfile`.
+
+```groovy
+gctsCloneRepository(
+  script: this,
+  host: "https://abap.server.com:port",
+  client: "000",
+  abapCredentialsId: 'ABAPUserPasswordCredentialsId',
+  repository: "myrepo"
+)
+```
+
+Example for the use in a YAML configuration file (such as `.pipeline/config.yaml`).
+
+```yaml
+steps:
+  <...>
+  gctsCloneRepository:
+    host: "https://abap.server.com:port"
+    client: "000"
+    username: "ABAPUsername"
+    password: "ABAPPassword"
+    repository: "myrepo"
+```

resources/metadata/gctsCloneRepository.yaml

@@ -0,0 +1,55 @@
+metadata:
+  name: gctsCloneRepository
+  description: Clones a Git repository
+  longDescription: |
+    Clones a Git repository from a remote repository to a local repository on an ABAP system. To be able to execute this step, the corresponding local repository has to exist on the local ABAP system.
+
+spec:
+  inputs:
+    secrets:
+      - name: abapCredentialsId
+        description: Jenkins credentials ID containing username and password for authentication to the ABAP system on which you want to clone the repository
+        type: jenkins
+    params:
+      - name: username
+        type: string
+        description: User to authenticate to the ABAP system
+        scope:
+        - PARAMETERS
+        - STAGES
+        - STEPS
+        mandatory: true
+        secret: true
+      - name: password
+        type: string
+        description: Password to authenticate to the ABAP system
+        scope:
+        - PARAMETERS
+        - STAGES
+        - STEPS
+        mandatory: true
+        secret: true
+      - name: repository
+        type: string
+        description: Specifies the name (ID) of the local repsitory on the ABAP system
+        scope:
+        - PARAMETERS
+        - STAGES
+        - STEPS
+        mandatory: true
+      - name: host
+        type: string
+        description: Specifies the protocol and host adress, including the port. Please provide in the format '<protocol>://<host>:<port>'
+        scope:
+        - PARAMETERS
+        - STAGES
+        - STEPS
+        mandatory: true
+      - name: client
+        type: string
+        description: Specifies the client of the ABAP system to be adressed
+        scope:
+        - PARAMETERS
+        - STAGES
+        - STEPS
+        mandatory: true

test/groovy/CommonStepsTest.groovy

@@ -136,6 +136,7 @@ public class CommonStepsTest extends BasePiperTest{
         'abapEnvironmentRunATCCheck', //implementing new golang pattern without fields
         'sonarExecuteScan', //implementing new golang pattern without fields
         'gctsCreateRepository', //implementing new golang pattern without fields
+        'gctsCloneRepository', //implementing new golang pattern without fields
         'fortifyExecuteScan', //implementing new golang pattern without fields
         'gctsDeploy', //implementing new golang pattern without fields
         'containerSaveImage', //implementing new golang pattern without fields

vars/gctsCloneRepository.groovy

@@ -0,0 +1,11 @@
+import groovy.transform.Field
+
+@Field String STEP_NAME = getClass().getName()
+@Field String METADATA_FILE = 'metadata/gctsCloneRepository.yaml'
+
+void call(Map parameters = [:]) {
+        List credentials = [
+        [type: 'usernamePassword', id: 'abapCredentialsId', env: ['PIPER_username', 'PIPER_password']]
+        ]
+        piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials)
+}