Retry capabilities for HTTP requests + enablement for Checkmarx step (#2346)

cmd/checkmarxExecuteScan.go

@@ -26,6 +26,8 @@ import (
 
 func checkmarxExecuteScan(config checkmarxExecuteScanOptions, telemetryData *telemetry.CustomData, influx *checkmarxExecuteScanInflux) {
 	client := &piperHttp.Client{}
+	options := piperHttp.ClientOptions{MaxRetries: config.MaxRetries}
+	client.SetOptions(options)
 	sys, err := checkmarx.NewSystemInstance(client, config.ServerURL, config.Username, config.Password)
 	if err != nil {
 		log.Entry().WithError(err).Fatalf("Failed to create Checkmarx client talking to URL %v", config.ServerURL)

cmd/checkmarxExecuteScan_generated.go

@@ -22,6 +22,7 @@ type checkmarxExecuteScanOptions struct {
 	FullScansScheduled            bool   `json:"fullScansScheduled,omitempty"`
 	GeneratePdfReport             bool   `json:"generatePdfReport,omitempty"`
 	Incremental                   bool   `json:"incremental,omitempty"`
+	MaxRetries                    int    `json:"maxRetries,omitempty"`
 	Password                      string `json:"password,omitempty"`
 	Preset                        string `json:"preset,omitempty"`
 	ProjectName                   string `json:"projectName,omitempty"`
@@ -235,6 +236,7 @@ func addCheckmarxExecuteScanFlags(cmd *cobra.Command, stepConfig *checkmarxExecu
 	cmd.Flags().BoolVar(&stepConfig.FullScansScheduled, "fullScansScheduled", true, "Whether full scans are to be scheduled or not. Should be used in relation with `incremental` and `fullScanCycle`")
 	cmd.Flags().BoolVar(&stepConfig.GeneratePdfReport, "generatePdfReport", true, "Whether to generate a PDF report of the analysis results or not")
 	cmd.Flags().BoolVar(&stepConfig.Incremental, "incremental", true, "Whether incremental scans are to be applied which optimizes the scan time but might reduce detection capabilities. Therefore full scans are still required from time to time and should be scheduled via `fullScansScheduled` and `fullScanCycle`")
+	cmd.Flags().IntVar(&stepConfig.MaxRetries, "maxRetries", 3, "Maximum number of HTTP request retries upon intermittend connetion interrupts")
 	cmd.Flags().StringVar(&stepConfig.Password, "password", os.Getenv("PIPER_password"), "The password to authenticate")
 	cmd.Flags().StringVar(&stepConfig.Preset, "preset", os.Getenv("PIPER_preset"), "The preset to use for scanning, if not set explicitly the step will attempt to look up the project's setting based on the availability of `checkmarxCredentialsId`")
 	cmd.Flags().StringVar(&stepConfig.ProjectName, "projectName", os.Getenv("PIPER_projectName"), "The name of the Checkmarx project to scan into")
@@ -316,6 +318,14 @@ func checkmarxExecuteScanMetadata() config.StepData {
 						Mandatory:   false,
 						Aliases:     []config.Alias{},
 					},
+					{
+						Name:        "maxRetries",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "int",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+					},
 					{
 						Name: "password",
 						ResourceRef: []config.ResourceReference{

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/google/uuid v1.1.2
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.0 // indirect
-	github.com/hashicorp/go-retryablehttp v0.6.7 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.7
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/vault/api v1.0.4
 	github.com/huandu/xstrings v1.3.2 // indirect

pkg/http/http.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/motemen/go-nuts/roundtime"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -25,6 +26,7 @@ import (
 // Client defines an http client object
 type Client struct {
 	maxRequestDuration        time.Duration
+	maxRetries                int
 	transportTimeout          time.Duration
 	transportSkipVerification bool
 	username                  string
@@ -43,6 +45,7 @@ type ClientOptions struct {
 	// for the request will be enforced. This should only be used if the
 	// length of the request bodies is known.
 	MaxRequestDuration time.Duration
+	MaxRetries         int
 	// TransportTimeout defaults to 3 minutes, if not specified. It is
 	// used for the transport layer and duration of handshakes and such.
 	TransportTimeout          time.Duration
@@ -196,6 +199,7 @@ func (c *Client) SetOptions(options ClientOptions) {
 	c.username = options.Username
 	c.password = options.Password
 	c.token = options.Token
+	c.maxRetries = options.MaxRetries
 
 	if options.Logger != nil {
 		c.logger = options.Logger
@@ -224,14 +228,26 @@ func (c *Client) initialize() *http.Client {
 		doLogRequestBodyOnDebug:  c.doLogRequestBodyOnDebug,
 		doLogResponseBodyOnDebug: c.doLogResponseBodyOnDebug,
 	}
-	var httpClient = &http.Client{
+
-		Timeout:   c.maxRequestDuration,
+	var httpClient *http.Client
-		Transport: transport,
+	if c.maxRetries > 0 {
-		Jar:       c.cookieJar,
+		retryClient := retryablehttp.NewClient()
+		retryClient.HTTPClient.Timeout = c.maxRequestDuration
+		retryClient.HTTPClient.Jar = c.cookieJar
+		retryClient.HTTPClient.Transport = transport
+		retryClient.RetryMax = c.maxRetries
+		httpClient = retryClient.StandardClient()
+	} else {
+		httpClient = &http.Client{}
+		httpClient.Timeout = c.maxRequestDuration
+		httpClient.Jar = c.cookieJar
+		httpClient.Transport = transport
 	}
+
 	if c.transportSkipVerification {
 		c.logger.Debugf("TLS verification disabled")
 	}
+
 	c.logger.Debugf("Transport timeout: %v, max request duration: %v", c.transportTimeout, c.maxRequestDuration)
 
 	return httpClient

pkg/http/http_test.go

@@ -303,6 +303,32 @@ func TestTransportSkipVerification(t *testing.T) {
 	}
 }
 
+func TestMaxRetries(t *testing.T) {
+	testCases := []struct {
+		client       Client
+		countedCalls int
+	}{
+		{client: Client{maxRetries: 0}, countedCalls: 1},
+		{client: Client{maxRetries: 2}, countedCalls: 3},
+		{client: Client{maxRetries: 3}, countedCalls: 4},
+	}
+
+	for _, testCase := range testCases {
+		// init
+		count := 0
+		svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			count++
+			w.WriteHeader(500)
+		}))
+		defer svr.Close()
+		// test
+		_, err := testCase.client.SendRequest(http.MethodGet, svr.URL, &bytes.Buffer{}, nil, nil)
+		// assert
+		assert.Error(t, err)
+		assert.Equal(t, testCase.countedCalls, count)
+	}
+}
+
 func TestParseHTTPResponseBodyJSON(t *testing.T) {
 
 	type myJSONStruct struct {

resources/metadata/checkmarx.yaml

@@ -72,6 +72,14 @@ spec:
           - STAGES
           - STEPS
         default: true
+      - name: maxRetries
+        type: int
+        description: Maximum number of HTTP request retries upon intermittend connetion interrupts
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        default: 3
       - name: password
         type: string
         description: The password to authenticate