ApiProxyDownload Command (#3197)

* ApiProxyDownload Command

* Lint Fixes

* Lint Fixes

* codereview fixes

* Code Review Fixes

* CodeReview Fixes

* CodeReview Fixes

* Code Review Fixes

* Code Review Changes

* CodeReview Fixes

* CodeReview Fixes

* CodeReview Fix

* CodeReview Fixes

* CodeReviw Fixes

* CodeReview Changes

* CodeReview Fixes

Co-authored-by: Oliver Feldmann <oliver.feldmann@sap.com>

cmd/apiProxyDownload.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/SAP/jenkins-library/pkg/cpi"
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/pkg/errors"
+)
+
+func apiProxyDownload(config apiProxyDownloadOptions, telemetryData *telemetry.CustomData) {
+	// Utils can be used wherever the command.ExecRunner interface is expected.
+	// It can also be used for example as a mavenExecRunner.
+	httpClient := &piperhttp.Client{}
+
+	// For HTTP calls import  piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	// and use a  &piperhttp.Client{} in a custom system
+	// Example: step checkmarxExecuteScan.go
+
+	// Error situations should be bubbled up until they reach the line below which will then stop execution
+	// through the log.Entry().Fatal() call leading to an os.Exit(1) in the end.
+	err := runApiProxyDownload(&config, telemetryData, httpClient)
+	if err != nil {
+		log.Entry().WithError(err).Fatal("step execution failed")
+	}
+}
+
+func runApiProxyDownload(config *apiProxyDownloadOptions, telemetryData *telemetry.CustomData, httpClient piperhttp.Sender) error {
+	clientOptions := piperhttp.ClientOptions{}
+	header := make(http.Header)
+	header.Add("Accept", "application/zip")
+	serviceKey, err := cpi.ReadCpiServiceKey(config.APIServiceKey)
+	if err != nil {
+		return err
+	}
+	downloadArtifactURL := fmt.Sprintf("%s/apiportal/api/1.0/Transport.svc/APIProxies?name=%s", serviceKey.OAuth.Host, config.APIProxyName)
+	tokenParameters := cpi.TokenParameters{TokenURL: serviceKey.OAuth.OAuthTokenProviderURL,
+		Username: serviceKey.OAuth.ClientID, Password: serviceKey.OAuth.ClientSecret, Client: httpClient}
+	token, err := cpi.CommonUtils.GetBearerToken(tokenParameters)
+	if err != nil {
+		return errors.Wrap(err, "failed to fetch Bearer Token")
+	}
+	clientOptions.Token = fmt.Sprintf("Bearer %s", token)
+	httpClient.SetOptions(clientOptions)
+	httpMethod := http.MethodGet
+	downloadResp, httpErr := httpClient.SendRequest(httpMethod, downloadArtifactURL, nil, header, nil)
+	if httpErr != nil {
+		return errors.Wrapf(httpErr, "HTTP %v request to %v failed with error", httpMethod, downloadArtifactURL)
+	}
+	if downloadResp == nil {
+		return errors.Errorf("did not retrieve a HTTP response: %v", httpErr)
+	}
+	failureMessage := "Failed to download API Proxy artefact"
+	httpFileDownloadRequestParameters := cpi.HttpFileDownloadRequestParameters{ErrMessage: failureMessage, FileDownloadPath: config.DownloadPath, Response: downloadResp}
+	return cpi.HttpCPIUtils.HandleHTTPFileDownloadResponse(httpFileDownloadRequestParameters)
+}

cmd/apiProxyDownload_generated.go

@@ -0,0 +1,170 @@
+// Code generated by piper's step-generator. DO NOT EDIT.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/SAP/jenkins-library/pkg/config"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/splunk"
+	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/SAP/jenkins-library/pkg/validation"
+	"github.com/spf13/cobra"
+)
+
+type apiProxyDownloadOptions struct {
+	APIServiceKey string `json:"apiServiceKey,omitempty"`
+	APIProxyName  string `json:"apiProxyName,omitempty"`
+	DownloadPath  string `json:"downloadPath,omitempty"`
+}
+
+// ApiProxyDownloadCommand Download a specific API Proxy from the API Portal
+func ApiProxyDownloadCommand() *cobra.Command {
+	const STEP_NAME = "apiProxyDownload"
+
+	metadata := apiProxyDownloadMetadata()
+	var stepConfig apiProxyDownloadOptions
+	var startTime time.Time
+	var logCollector *log.CollectorHook
+
+	var createApiProxyDownloadCmd = &cobra.Command{
+		Use:   STEP_NAME,
+		Short: "Download a specific API Proxy from the API Portal",
+		Long:  `With this step you can download a specific API Proxy from the API Portal, which returns a zip file with the api proxy contents in to current workspace using the OData API. Learn more about the SAP API Management API for downloading an api proxy artifact [here](https://help.sap.com/viewer/66d066d903c2473f81ec33acfe2ccdb4/Cloud/en-US/e26b3320cd534ae4bc743af8013a8abb.html).`,
+		PreRunE: func(cmd *cobra.Command, _ []string) error {
+			startTime = time.Now()
+			log.SetStepName(STEP_NAME)
+			log.SetVerbose(GeneralConfig.Verbose)
+
+			GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens)
+
+			path, _ := os.Getwd()
+			fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path}
+			log.RegisterHook(fatalHook)
+
+			err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile)
+			if err != nil {
+				log.SetErrorCategory(log.ErrorConfiguration)
+				return err
+			}
+			log.RegisterSecret(stepConfig.APIServiceKey)
+
+			if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 {
+				sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID)
+				log.RegisterHook(&sentryHook)
+			}
+
+			if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+				logCollector = &log.CollectorHook{CorrelationID: GeneralConfig.CorrelationID}
+				log.RegisterHook(logCollector)
+			}
+
+			validation, err := validation.New(validation.WithJSONNamesForStructFields(), validation.WithPredefinedErrorMessages())
+			if err != nil {
+				return err
+			}
+			if err = validation.ValidateStruct(stepConfig); err != nil {
+				log.SetErrorCategory(log.ErrorConfiguration)
+				return err
+			}
+
+			return nil
+		},
+		Run: func(_ *cobra.Command, _ []string) {
+			telemetryData := telemetry.CustomData{}
+			telemetryData.ErrorCode = "1"
+			handler := func() {
+				config.RemoveVaultSecretFiles()
+				telemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds())
+				telemetryData.ErrorCategory = log.GetErrorCategory().String()
+				telemetry.Send(&telemetryData)
+				if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+					splunk.Send(&telemetryData, logCollector)
+				}
+			}
+			log.DeferExitHandler(handler)
+			defer handler()
+			telemetry.Initialize(GeneralConfig.NoTelemetry, STEP_NAME)
+			if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+				splunk.Initialize(GeneralConfig.CorrelationID,
+					GeneralConfig.HookConfig.SplunkConfig.Dsn,
+					GeneralConfig.HookConfig.SplunkConfig.Token,
+					GeneralConfig.HookConfig.SplunkConfig.Index,
+					GeneralConfig.HookConfig.SplunkConfig.SendLogs)
+			}
+			apiProxyDownload(stepConfig, &telemetryData)
+			telemetryData.ErrorCode = "0"
+			log.Entry().Info("SUCCESS")
+		},
+	}
+
+	addApiProxyDownloadFlags(createApiProxyDownloadCmd, &stepConfig)
+	return createApiProxyDownloadCmd
+}
+
+func addApiProxyDownloadFlags(cmd *cobra.Command, stepConfig *apiProxyDownloadOptions) {
+	cmd.Flags().StringVar(&stepConfig.APIServiceKey, "apiServiceKey", os.Getenv("PIPER_apiServiceKey"), "Service key JSON string to access the API Management Runtime service instance of plan 'api'")
+	cmd.Flags().StringVar(&stepConfig.APIProxyName, "apiProxyName", os.Getenv("PIPER_apiProxyName"), "Specifies the name of the API Proxy.")
+	cmd.Flags().StringVar(&stepConfig.DownloadPath, "downloadPath", os.Getenv("PIPER_downloadPath"), "Specifies api proxy download directory location. The file name should not be included in the path.")
+
+	cmd.MarkFlagRequired("apiServiceKey")
+	cmd.MarkFlagRequired("apiProxyName")
+	cmd.MarkFlagRequired("downloadPath")
+}
+
+// retrieve step metadata
+func apiProxyDownloadMetadata() config.StepData {
+	var theMetaData = config.StepData{
+		Metadata: config.StepMetadata{
+			Name:        "apiProxyDownload",
+			Aliases:     []config.Alias{},
+			Description: "Download a specific API Proxy from the API Portal",
+		},
+		Spec: config.StepSpec{
+			Inputs: config.StepInputs{
+				Secrets: []config.StepSecrets{
+					{Name: "apimApiServiceKeyCredentialsId", Description: "Jenkins secret text credential ID containing the service key to the API Management Runtime service instance of plan 'api'", Type: "jenkins"},
+				},
+				Parameters: []config.StepParameters{
+					{
+						Name: "apiServiceKey",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name:  "apimApiServiceKeyCredentialsId",
+								Param: "apiServiceKey",
+								Type:  "secret",
+							},
+						},
+						Scope:     []string{"PARAMETERS"},
+						Type:      "string",
+						Mandatory: true,
+						Aliases:   []config.Alias{},
+						Default:   os.Getenv("PIPER_apiServiceKey"),
+					},
+					{
+						Name:        "apiProxyName",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   true,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_apiProxyName"),
+					},
+					{
+						Name:        "downloadPath",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   true,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_downloadPath"),
+					},
+				},
+			},
+		},
+	}
+	return theMetaData
+}

cmd/apiProxyDownload_generated_test.go

@@ -0,0 +1,17 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestApiProxyDownloadCommand(t *testing.T) {
+	t.Parallel()
+
+	testCmd := ApiProxyDownloadCommand()
+
+	// only high level testing performed - details are tested in step generation procedure
+	assert.Equal(t, "apiProxyDownload", testCmd.Use, "command name incorrect")
+
+}

cmd/apiProxyDownload_test.go

@@ -0,0 +1,75 @@
+package cmd
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/SAP/jenkins-library/pkg/mock"
+	"github.com/stretchr/testify/assert"
+)
+
+type apiProxyDownloadMockUtils struct {
+	*mock.ExecMockRunner
+	*mock.FilesMock
+}
+
+func TestRunApiProxyDownload(t *testing.T) {
+	t.Parallel()
+
+	t.Run("Successfull Download of API Proxy", func(t *testing.T) {
+		tempDir, tmpErr := ioutil.TempDir("", "")
+		defer os.RemoveAll(tempDir) // clean up
+		if tmpErr != nil {
+			t.Fatal("Failed to create temporary directory")
+		}
+		apiServiceKey := `{
+			"oauth": {
+				"url": "https://demo",
+				"clientid": "demouser",
+				"clientsecret": "******",
+				"tokenurl": "https://demo/oauth/token"
+			}
+		}`
+		config := apiProxyDownloadOptions{
+			APIServiceKey: apiServiceKey,
+			APIProxyName:  "flow1",
+			DownloadPath:  tempDir,
+		}
+		httpClient := httpMockCpis{CPIFunction: "APIProxyDownload", ResponseBody: ``, TestType: "PositiveAndGetetIntegrationArtifactDownloadResBody"}
+		err := runApiProxyDownload(&config, nil, &httpClient)
+		absolutePath := filepath.Join(tempDir, "flow1.zip")
+		if assert.NoError(t, err) {
+			t.Run("check file", func(t *testing.T) {
+				assert.Equal(t, fileExists(absolutePath), true)
+			})
+			t.Run("check url", func(t *testing.T) {
+				assert.Equal(t, "https://demo/apiportal/api/1.0/Transport.svc/APIProxies?name=flow1", httpClient.URL)
+			})
+
+			t.Run("check method", func(t *testing.T) {
+				assert.Equal(t, "GET", httpClient.Method)
+			})
+		}
+	})
+
+	t.Run("Failed case of api proxy artifact Download", func(t *testing.T) {
+		apiServiceKey := `{
+			"oauth": {
+				"url": "https://demo",
+				"clientid": "demouser",
+				"clientsecret": "******",
+				"tokenurl": "https://demo/oauth/token"
+			}
+		}`
+		config := apiProxyDownloadOptions{
+			APIServiceKey: apiServiceKey,
+			APIProxyName:  "proxy1",
+			DownloadPath:  "tmp",
+		}
+		httpClient := httpMockCpis{CPIFunction: "APIProxyDownloadFailure", ResponseBody: ``, TestType: "Negative"}
+		err := runApiProxyDownload(&config, nil, &httpClient)
+		assert.EqualError(t, err, "HTTP GET request to https://demo/apiportal/api/1.0/Transport.svc/APIProxies?name=proxy1 failed with error: Service not Found")
+	})
+}

cmd/metadata_generated.go

@@ -22,6 +22,7 @@ func GetAllStepMetadata() map[string]config.StepData {
 		"abapEnvironmentPullGitRepo":                abapEnvironmentPullGitRepoMetadata(),
 		"abapEnvironmentRunATCCheck":                abapEnvironmentRunATCCheckMetadata(),
 		"abapEnvironmentRunAUnitTest":               abapEnvironmentRunAUnitTestMetadata(),
+		"apiProxyDownload":                          apiProxyDownloadMetadata(),
 		"batsExecuteTests":                          batsExecuteTestsMetadata(),
 		"checkmarxExecuteScan":                      checkmarxExecuteScanMetadata(),
 		"cloudFoundryCreateService":                 cloudFoundryCreateServiceMetadata(),

cmd/piper.go

@@ -166,6 +166,7 @@ func Execute() {
 	rootCmd.AddCommand(InfluxWriteDataCommand())
 	rootCmd.AddCommand(AbapEnvironmentRunAUnitTestCommand())
 	rootCmd.AddCommand(CheckStepActiveCommand())
+	rootCmd.AddCommand(ApiProxyDownloadCommand())
 
 	addRootFlags(rootCmd)
 

documentation/docs/steps/apiProxyDownload.md

@@ -0,0 +1,30 @@
+# ${docGenStepName}
+
+## ${docGenDescription}
+
+## Prerequisites
+
+## ${docGenParameters}
+
+## ${docGenConfiguration}
+
+## ${docJenkinsPluginDependencies}
+
+## Example
+
+Example configuration for the use in a `Jenkinsfile`.
+
+```groovy
+apiProxyDownload script: this
+```
+
+Example for the use in a YAML configuration file (such as `.pipeline/config.yaml`).
+
+```yaml
+steps:
+  <...>
+  apiProxyDownload:
+    apimApiServiceKeyCredentialsId: 'MY_API_SERVICE_KEY'
+    apiProxyName: 'MY_API_PROXY_NAME'
+    downloadPath: MY_API_PROXY_DOWNLOAD_PATH
+```

documentation/mkdocs.yml

@@ -64,6 +64,7 @@ nav:
         - abapEnvironmentCreateSystem: steps/abapEnvironmentCreateSystem.md
         - abapEnvironmentPullGitRepo: steps/abapEnvironmentPullGitRepo.md
         - abapEnvironmentRunATCCheck: steps/abapEnvironmentRunATCCheck.md
+        - apiProxyDownload: steps/apiProxyDownload.md
         - artifactPrepareVersion: steps/artifactPrepareVersion.md
         - batsExecuteTests: steps/batsExecuteTests.md
         - buildExecute: steps/buildExecute.md

pkg/cpi/commonUtils.go

@@ -3,9 +3,14 @@ package cpi
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/SAP/jenkins-library/pkg/log"
+	"io"
 	"io/ioutil"
+	"mime"
 	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/SAP/jenkins-library/pkg/log"
 
 	"github.com/Jeffail/gabs/v2"
 	piperhttp "github.com/SAP/jenkins-library/pkg/http"
@@ -17,12 +22,23 @@ type CommonUtils interface {
 	GetBearerToken() (string, error)
 }
 
+//HttpCPIUtils for CPI
+type HttpCPIUtils interface {
+	HandleHTTPFileDownloadResponse() error
+}
+
 //TokenParameters struct
 type TokenParameters struct {
 	TokenURL, Username, Password string
 	Client                       piperhttp.Sender
 }
 
+//HttpParameters struct
+type HttpFileDownloadRequestParameters struct {
+	ErrMessage, FileDownloadPath string
+	Response                     *http.Response
+}
+
 // ServiceKey contains information about a CPI service key
 type ServiceKey struct {
 	OAuth OAuth `json:"oauth"`
@@ -91,3 +107,44 @@ func (tokenParameters TokenParameters) GetBearerToken() (string, error) {
 	token := jsonResponse.Path("access_token").Data().(string)
 	return token, nil
 }
+
+// HandleHTTPFileDownloadResponse - Handle the file download response for http multipart response
+func (httpFileDownloadRequestParameters HttpFileDownloadRequestParameters) HandleHTTPFileDownloadResponse() error {
+	response := httpFileDownloadRequestParameters.Response
+	contentDisposition := response.Header.Get("Content-Disposition")
+	disposition, params, err := mime.ParseMediaType(contentDisposition)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read filename from http response headers, Content-Disposition %s", disposition)
+	}
+	filename := params["filename"]
+
+	if response != nil && response.Body != nil {
+		defer response.Body.Close()
+	}
+
+	if response.StatusCode == 200 {
+		workspaceRelativePath := httpFileDownloadRequestParameters.FileDownloadPath
+		err = os.MkdirAll(workspaceRelativePath, 0755)
+		// handling error while creating a workspce directoy for file download, if one not exist already!
+		if err != nil {
+			return errors.Wrapf(err, "Failed to create workspace directory")
+		}
+		zipFileName := filepath.Join(workspaceRelativePath, filename)
+		file, err := os.Create(zipFileName)
+		// handling error while creating a file in the filesystem
+		if err != nil {
+			return errors.Wrap(err, "failed to create zip archive of api proxy")
+		}
+		_, err = io.Copy(file, response.Body)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	responseBody, readErr := ioutil.ReadAll(response.Body)
+	if readErr != nil {
+		return errors.Wrapf(readErr, "HTTP response body could not be read, Response status code: %v", response.StatusCode)
+	}
+	log.Entry().Errorf("a HTTP error occurred! Response body: %v, Response status code : %v", responseBody, response.StatusCode)
+	return errors.Errorf("%s, Response Status code: %v", httpFileDownloadRequestParameters.ErrMessage, response.StatusCode)
+}

pkg/cpi/mockingUtils.go

@@ -30,7 +30,7 @@ func GetCPIFunctionMockResponse(functionName, testType string) (*http.Response,
 		return GetIntegrationArtifactGetMplStatusCommandMockResponse(testType)
 	case "IntegrationArtifactGetServiceEndpoint":
 		return GetIntegrationArtifactGetServiceEndpointCommandMockResponse(testType)
-	case "IntegrationArtifactDownload":
+	case "IntegrationArtifactDownload", "APIProxyDownload":
 		return IntegrationArtifactDownloadCommandMockResponse(testType)
 	case "GetIntegrationDesigntimeArtifact":
 		return GetIntegrationDesigntimeArtifactMockResponse(testType)

resources/metadata/apiProxyDownload.yaml

@@ -0,0 +1,40 @@
+metadata:
+  name: apiProxyDownload
+  description: Download a specific API Proxy from the API Portal
+  longDescription: |
+    With this step you can download a specific API Proxy from the API Portal, which returns a zip file with the api proxy contents in to current workspace using the OData API. Learn more about the SAP API Management API for downloading an api proxy artifact [here](https://help.sap.com/viewer/66d066d903c2473f81ec33acfe2ccdb4/Cloud/en-US/e26b3320cd534ae4bc743af8013a8abb.html).
+
+spec:
+  inputs:
+    secrets:
+      - name: apimApiServiceKeyCredentialsId
+        description: Jenkins secret text credential ID containing the service key to the API Management Runtime service instance of plan 'api'
+        type: jenkins
+    params:
+      - name: apiServiceKey
+        type: string
+        description: Service key JSON string to access the API Management Runtime service instance of plan 'api'
+        scope:
+          - PARAMETERS
+        mandatory: true
+        secret: true
+        resourceRef:
+          - name: apimApiServiceKeyCredentialsId
+            type: secret
+            param: apiServiceKey
+      - name: apiProxyName
+        type: string
+        description: Specifies the name of the API Proxy.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        mandatory: true
+      - name: downloadPath
+        type: string
+        description: Specifies api proxy download directory location. The file name should not be included in the path.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        mandatory: true

test/groovy/CommonStepsTest.groovy

@@ -200,6 +200,7 @@ public class CommonStepsTest extends BasePiperTest{
         'readPipelineEnv', //implementing new golang pattern without fields
         'transportRequestUploadCTS', //implementing new golang pattern without fields
         'isChangeInDevelopment', //implementing new golang pattern without fields
+        'apiProxyDownload', //implementing new golang pattern without fields
     ]
 
     @Test

vars/apiProxyDownload.groovy

@@ -0,0 +1,11 @@
+import groovy.transform.Field
+
+@Field String STEP_NAME = getClass().getName()
+@Field String METADATA_FILE = 'metadata/apiProxyDownload.yaml'
+
+void call(Map parameters = [:]) {
+    List credentials = [
+        [type: 'token', id: 'apimApiServiceKeyCredentialsId', env: ['PIPER_apiServiceKey']]
+    ]
+    piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials)
+}