fix(codeqlExecuteScan): pagination call for getting codescanning results (#4370)

pagination call for getting code scanning results

---------

Co-authored-by: sumeet patil <sumeet.patil@sap.com>

cmd/codeqlExecuteScan.go

@@ -280,25 +280,25 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem
 			return reports, err
 		}
 
-		if config.CheckForCompliance {
-			codeqlScanAuditInstance := codeql.NewCodeqlScanAuditInstance(repoInfo.serverUrl, repoInfo.owner, repoInfo.repo, token, []string{})
-			scanResults, err := codeqlScanAuditInstance.GetVulnerabilities(repoInfo.ref)
-			if err != nil {
-				return reports, errors.Wrap(err, "failed to get scan results")
-			}
+		codeqlScanAuditInstance := codeql.NewCodeqlScanAuditInstance(repoInfo.serverUrl, repoInfo.owner, repoInfo.repo, token, []string{})
+		scanResults, err := codeqlScanAuditInstance.GetVulnerabilities(repoInfo.ref)
+		if err != nil {
+			return reports, errors.Wrap(err, "failed to get scan results")
+		}
 
+		codeqlAudit := codeql.CodeqlAudit{ToolName: "codeql", RepositoryUrl: repoUrl, CodeScanningLink: repoCodeqlScanUrl, RepositoryReferenceUrl: repoReference, ScanResults: scanResults}
+		paths, err := codeql.WriteJSONReport(codeqlAudit, config.ModulePath)
+		if err != nil {
+			return reports, errors.Wrap(err, "failed to write json compliance report")
+		}
+
+		if config.CheckForCompliance {
 			unaudited := (scanResults.Total - scanResults.Audited)
 			if unaudited > config.VulnerabilityThresholdTotal {
 				msg := fmt.Sprintf("Your repository %v with ref %v is not compliant. Total unaudited issues are %v which is greater than the VulnerabilityThresholdTotal count %v", repoUrl, repoInfo.ref, unaudited, config.VulnerabilityThresholdTotal)
 				return reports, errors.Errorf(msg)
 			}
 
-			codeqlAudit := codeql.CodeqlAudit{ToolName: "codeql", RepositoryUrl: repoUrl, CodeScanningLink: repoCodeqlScanUrl, RepositoryReferenceUrl: repoReference, ScanResults: scanResults}
-			paths, err := codeql.WriteJSONReport(codeqlAudit, config.ModulePath)
-			if err != nil {
-				return reports, errors.Wrap(err, "failed to write json compliance report")
-			}
-
 			reports = append(reports, paths...)
 		}
 	}

pkg/codeql/codeql.go

@@ -2,6 +2,7 @@ package codeql
 
 import (
 	"context"
+	"errors"
 
 	sapgithub "github.com/SAP/jenkins-library/pkg/github"
 	"github.com/google/go-github/v45/github"
@@ -13,9 +14,11 @@ type CodeqlScanAudit interface {
 
 type githubCodeqlScanningService interface {
 	ListAlertsForRepo(ctx context.Context, owner, repo string, opts *github.AlertListOptions) ([]*github.Alert, *github.Response, error)
+	ListAnalysesForRepo(ctx context.Context, owner, repo string, opts *github.AnalysesListOptions) ([]*github.ScanningAnalysis, *github.Response, error)
 }
 
-const auditStateOpen = "open"
+const auditStateOpen string = "open"
+const perPageCount int = 100
 
 func NewCodeqlScanAuditInstance(serverUrl, owner, repository, token string, trustedCerts []string) CodeqlScanAuditInstance {
 	return CodeqlScanAuditInstance{serverUrl: serverUrl, owner: owner, repository: repository, token: token, trustedCerts: trustedCerts}
@@ -36,32 +39,67 @@ func (codeqlScanAudit *CodeqlScanAuditInstance) GetVulnerabilities(analyzedRef s
 	if err != nil {
 		return CodeqlScanning{}, err
 	}
+	totalAlerts, err := getTotalAlertsFromClient(ctx, client.CodeScanning, analyzedRef, codeqlScanAudit)
 
-	return getVulnerabilitiesFromClient(ctx, client.CodeScanning, analyzedRef, codeqlScanAudit)
+	return getVulnerabilitiesFromClient(ctx, client.CodeScanning, analyzedRef, codeqlScanAudit, totalAlerts)
 }
 
-func getVulnerabilitiesFromClient(ctx context.Context, codeScanning githubCodeqlScanningService, analyzedRef string, codeqlScanAudit *CodeqlScanAuditInstance) (CodeqlScanning, error) {
-	alertOptions := github.AlertListOptions{
-		State:       "",
-		Ref:         analyzedRef,
-		ListOptions: github.ListOptions{},
+func getTotalAlertsFromClient(ctx context.Context, codeScannning githubCodeqlScanningService, analyzedRef string, codeqlScanAudit *CodeqlScanAuditInstance) (int, error) {
+	analysesOptions := github.AnalysesListOptions{
+		Ref: &analyzedRef,
 	}
-
-	alerts, _, err := codeScanning.ListAlertsForRepo(ctx, codeqlScanAudit.owner, codeqlScanAudit.repository, &alertOptions)
+	analyses, _, err := codeScannning.ListAnalysesForRepo(ctx, codeqlScanAudit.owner, codeqlScanAudit.repository, &analysesOptions)
 	if err != nil {
-		return CodeqlScanning{}, err
+		return 0, err
 	}
+	if len(analyses) < 1 {
+		return 0, errors.New("analyses for ref not found")
+	}
+	return *analyses[0].ResultsCount, nil
+}
 
-	openStateCount := 0
-	for _, alert := range alerts {
-		if *alert.State == auditStateOpen {
-			openStateCount = openStateCount + 1
-		}
+func getVulnerabilitiesFromClient(ctx context.Context, codeScanning githubCodeqlScanningService, analyzedRef string, codeqlScanAudit *CodeqlScanAuditInstance, totalAlerts int) (CodeqlScanning, error) {
+	pages := totalAlerts/perPageCount + 1
+	errChan := make(chan error)
+	openStateCountChan := make(chan int)
+	for page := 1; page <= pages; page++ {
+		go func(i int) {
+			alertOptions := github.AlertListOptions{
+				State: "",
+				Ref:   analyzedRef,
+				ListOptions: github.ListOptions{
+					Page:    i,
+					PerPage: perPageCount,
+				},
+			}
+
+			alerts, _, err := codeScanning.ListAlertsForRepo(ctx, codeqlScanAudit.owner, codeqlScanAudit.repository, &alertOptions)
+			if err != nil {
+				errChan <- err
+				return
+			}
+
+			openStateCount := 0
+			for _, alert := range alerts {
+				if *alert.State == auditStateOpen {
+					openStateCount = openStateCount + 1
+				}
+			}
+			openStateCountChan <- len(alerts) - openStateCount
+		}(page)
 	}
 
 	codeqlScanning := CodeqlScanning{}
-	codeqlScanning.Total = len(alerts)
-	codeqlScanning.Audited = (codeqlScanning.Total - openStateCount)
+	codeqlScanning.Total = totalAlerts
+	for i := 0; i < pages; i++ {
+		select {
+		case openStateCount := <-openStateCountChan:
+			codeqlScanning.Audited += openStateCount
+		case err := <-errChan:
+			return CodeqlScanning{}, err
+		}
+	}
+
 	return codeqlScanning, nil
 }
 

pkg/codeql/codeql_test.go

@@ -18,10 +18,43 @@ type githubCodeqlScanningMock struct {
 func (g *githubCodeqlScanningMock) ListAlertsForRepo(ctx context.Context, owner, repo string, opts *github.AlertListOptions) ([]*github.Alert, *github.Response, error) {
 	openState := "open"
 	closedState := "closed"
-	alerts := []*github.Alert{{State: &openState}, {State: &openState}, {State: &closedState}}
+	alerts := []*github.Alert{}
+
+	if repo == "testRepo1" {
+		alerts = append(alerts, &github.Alert{State: &openState})
+		alerts = append(alerts, &github.Alert{State: &openState})
+		alerts = append(alerts, &github.Alert{State: &closedState})
+	}
+
+	if repo == "testRepo2" {
+		if opts.Page == 1 {
+			for i := 0; i < 50; i++ {
+				alerts = append(alerts, &github.Alert{State: &openState})
+			}
+			for i := 0; i < 50; i++ {
+				alerts = append(alerts, &github.Alert{State: &closedState})
+			}
+		}
+
+		if opts.Page == 2 {
+			for i := 0; i < 10; i++ {
+				alerts = append(alerts, &github.Alert{State: &openState})
+			}
+			for i := 0; i < 30; i++ {
+				alerts = append(alerts, &github.Alert{State: &closedState})
+			}
+		}
+	}
+
 	return alerts, nil, nil
 }
 
+func (g *githubCodeqlScanningMock) ListAnalysesForRepo(ctx context.Context, owner, repo string, opts *github.AnalysesListOptions) ([]*github.ScanningAnalysis, *github.Response, error) {
+	resultsCount := 3
+	analysis := []*github.ScanningAnalysis{{ResultsCount: &resultsCount}}
+	return analysis, nil, nil
+}
+
 type githubCodeqlScanningErrorMock struct {
 }
 
@@ -29,22 +62,38 @@ func (g *githubCodeqlScanningErrorMock) ListAlertsForRepo(ctx context.Context, o
 	return []*github.Alert{}, nil, errors.New("Some error")
 }
 
+func (g *githubCodeqlScanningErrorMock) ListAnalysesForRepo(ctx context.Context, owner, repo string, opts *github.AnalysesListOptions) ([]*github.ScanningAnalysis, *github.Response, error) {
+	return []*github.ScanningAnalysis{}, nil, errors.New("Some error")
+}
+
 func TestGetVulnerabilitiesFromClient(t *testing.T) {
 	ctx := context.Background()
 	t.Parallel()
 	t.Run("Success", func(t *testing.T) {
 		ghCodeqlScanningMock := githubCodeqlScanningMock{}
-		codeqlScanAuditInstance := NewCodeqlScanAuditInstance("", "", "", "", []string{})
-		codeScanning, err := getVulnerabilitiesFromClient(ctx, &ghCodeqlScanningMock, "ref", &codeqlScanAuditInstance)
+		totalAlerts := 3
+		codeqlScanAuditInstance := NewCodeqlScanAuditInstance("", "", "testRepo1", "", []string{})
+		codeScanning, err := getVulnerabilitiesFromClient(ctx, &ghCodeqlScanningMock, "ref", &codeqlScanAuditInstance, totalAlerts)
 		assert.NoError(t, err)
 		assert.Equal(t, 3, codeScanning.Total)
 		assert.Equal(t, 1, codeScanning.Audited)
 	})
 
+	t.Run("Success with pagination results", func(t *testing.T) {
+		ghCodeqlScanningMock := githubCodeqlScanningMock{}
+		totalAlerts := 120
+		codeqlScanAuditInstance := NewCodeqlScanAuditInstance("", "", "testRepo2", "", []string{})
+		codeScanning, err := getVulnerabilitiesFromClient(ctx, &ghCodeqlScanningMock, "ref", &codeqlScanAuditInstance, totalAlerts)
+		assert.NoError(t, err)
+		assert.Equal(t, 120, codeScanning.Total)
+		assert.Equal(t, 80, codeScanning.Audited)
+	})
+
 	t.Run("Error", func(t *testing.T) {
 		ghCodeqlScanningErrorMock := githubCodeqlScanningErrorMock{}
+		totalAlerts := 3
 		codeqlScanAuditInstance := NewCodeqlScanAuditInstance("", "", "", "", []string{})
-		_, err := getVulnerabilitiesFromClient(ctx, &ghCodeqlScanningErrorMock, "ref", &codeqlScanAuditInstance)
+		_, err := getVulnerabilitiesFromClient(ctx, &ghCodeqlScanningErrorMock, "ref", &codeqlScanAuditInstance, totalAlerts)
 		assert.Error(t, err)
 	})
 }
@@ -58,3 +107,22 @@ func TestGetApiUrl(t *testing.T) {
 		assert.Equal(t, "https://github.test.org/api/v3", getApiUrl("https://github.test.org"))
 	})
 }
+
+func TestGetTotalAnalysesFromClient(t *testing.T) {
+	ctx := context.Background()
+	t.Parallel()
+	t.Run("Success", func(t *testing.T) {
+		ghCodeqlScanningMock := githubCodeqlScanningMock{}
+		codeqlScanAuditInstance := NewCodeqlScanAuditInstance("", "", "", "", []string{})
+		total, err := getTotalAlertsFromClient(ctx, &ghCodeqlScanningMock, "ref", &codeqlScanAuditInstance)
+		assert.NoError(t, err)
+		assert.Equal(t, 3, total)
+	})
+
+	t.Run("Error", func(t *testing.T) {
+		ghCodeqlScanningErrorMock := githubCodeqlScanningErrorMock{}
+		codeqlScanAuditInstance := NewCodeqlScanAuditInstance("", "", "", "", []string{})
+		_, err := getTotalAlertsFromClient(ctx, &ghCodeqlScanningErrorMock, "ref", &codeqlScanAuditInstance)
+		assert.Error(t, err)
+	})
+}