feat(codeql): added generating file toolrun_codeql.json (#4240)

* added generating toolrun file for codeql

cmd/codeqlExecuteScan.go

@@ -4,12 +4,14 @@ import (
 	"fmt"
 	"os"
 	"regexp"
+	"strings"
 
 	"github.com/SAP/jenkins-library/pkg/command"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/orchestrator"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/SAP/jenkins-library/pkg/toolrecord"
 	"github.com/pkg/errors"
 )
 
@@ -235,5 +237,100 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem
 		return err
 	}
 
+	// create toolrecord file
+	toolRecordFileName, err := createToolRecordCodeql(utils, "./", *config)
+	if err != nil {
+		// do not fail until the framework is well established
+		log.Entry().Warning("TR_CODEQL: Failed to create toolrecord file ...", err)
+	} else {
+		reports = append(reports, piperutils.Path{Target: toolRecordFileName})
+	}
+
 	return nil
 }
+
+func createToolRecordCodeql(utils codeqlExecuteScanUtils, workspace string, config codeqlExecuteScanOptions) (string, error) {
+	repoURL := strings.TrimSuffix(config.Repository, ".git")
+	toolInstance, orgName, repoName, err := parseRepositoryURL(repoURL)
+	if err != nil {
+		return "", err
+	}
+	record := toolrecord.New(utils, workspace, "codeql", toolInstance)
+	record.DisplayName = fmt.Sprintf("%s %s - %s %s", orgName, repoName, config.AnalyzedRef, config.CommitID)
+	record.DisplayURL = fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoURL, config.AnalyzedRef)
+	// Repository
+	err = record.AddKeyData("repository",
+		fmt.Sprintf("%s/%s", orgName, repoName),
+		fmt.Sprintf("%s %s", orgName, repoName),
+		config.Repository)
+	if err != nil {
+		return "", err
+	}
+	// Repository Reference
+	repoReference, err := buildRepoReference(repoURL, config.AnalyzedRef)
+	if err != nil {
+		log.Entry().WithError(err).Warn("Failed to build repository reference")
+	}
+	err = record.AddKeyData("repositoryReference",
+		config.AnalyzedRef,
+		fmt.Sprintf("%s - %s", repoName, config.AnalyzedRef),
+		repoReference)
+	if err != nil {
+		return "", err
+	}
+	// Scan Results
+	err = record.AddKeyData("scanResult",
+		fmt.Sprintf("%s/%s", config.AnalyzedRef, config.CommitID),
+		fmt.Sprintf("%s %s - %s %s", orgName, repoName, config.AnalyzedRef, config.CommitID),
+		fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoURL, config.AnalyzedRef))
+	if err != nil {
+		return "", err
+	}
+	err = record.Persist()
+	if err != nil {
+		return "", err
+	}
+	return record.GetFileName(), nil
+}
+
+func parseRepositoryURL(repository string) (toolInstance, orgName, repoName string, err error) {
+	if repository == "" {
+		err = errors.New("Repository param is not set")
+		return
+	}
+	fullRepo := strings.TrimSuffix(repository, ".git")
+	// regexp for toolInstance
+	re := regexp.MustCompile(`^[a-zA-Z0-9]+://[a-zA-Z0-9-_.]+/`)
+	matchedHost := re.FindAllString(fullRepo, -1)
+	if len(matchedHost) == 0 {
+		err = errors.New("Unable to parse tool instance from repository url")
+		return
+	}
+	orgRepoNames := strings.Split(strings.TrimPrefix(fullRepo, matchedHost[0]), "/")
+	if len(orgRepoNames) < 2 {
+		err = errors.New("Unable to parse organization and repo names from repository url")
+		return
+	}
+
+	toolInstance = strings.Trim(matchedHost[0], "/")
+	orgName = orgRepoNames[0]
+	repoName = orgRepoNames[1]
+	return
+}
+
+func buildRepoReference(repository, analyzedRef string) (string, error) {
+	if repository == "" || analyzedRef == "" {
+		return "", errors.New("Repository or analyzedRef param is not set")
+	}
+	ref := strings.Split(analyzedRef, "/")
+	if len(ref) < 3 {
+		return "", errors.New(fmt.Sprintf("Wrong analyzedRef format: %s", analyzedRef))
+	}
+	if strings.Contains(analyzedRef, "pull") {
+		if len(ref) < 4 {
+			return "", errors.New(fmt.Sprintf("Wrong analyzedRef format: %s", analyzedRef))
+		}
+		return fmt.Sprintf("%s/pull/%s", repository, ref[2]), nil
+	}
+	return fmt.Sprintf("%s/tree/%s", repository, ref[2]), nil
+}

cmd/codeqlExecuteScan_generated.go

@@ -45,6 +45,7 @@ func (p *codeqlExecuteScanReports) persist(stepConfig codeqlExecuteScanOptions,
 	content := []gcs.ReportOutputParam{
 		{FilePattern: "**/*.csv", ParamRef: "", StepResultType: "codeql"},
 		{FilePattern: "**/*.sarif", ParamRef: "", StepResultType: "codeql"},
+		{FilePattern: "**/toolrun_codeql_*.json", ParamRef: "", StepResultType: "codeql"},
 	}
 	envVars := []gcs.EnvVar{
 		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: gcpJsonKeyFilePath, Modified: false},
@@ -341,6 +342,7 @@ func codeqlExecuteScanMetadata() config.StepData {
 						Parameters: []map[string]interface{}{
 							{"filePattern": "**/*.csv", "type": "codeql"},
 							{"filePattern": "**/*.sarif", "type": "codeql"},
+							{"filePattern": "**/toolrun_codeql_*.json", "type": "codeql"},
 						},
 					},
 				},

cmd/codeqlExecuteScan_test.go

@@ -90,3 +90,174 @@ func TestGetGitRepoInfo(t *testing.T) {
 		assert.Error(t, getGitRepoInfo("github.hello.test/Testing/fortify", &repoInfo))
 	})
 }
+
+func TestParseRepositoryURL(t *testing.T) {
+	t.Run("Valid repository", func(t *testing.T) {
+		repository := "https://github.hello.test/Testing/fortify.git"
+		toolInstance, orgName, repoName, err := parseRepositoryURL(repository)
+		assert.NoError(t, err)
+		assert.Equal(t, "https://github.hello.test", toolInstance)
+		assert.Equal(t, "Testing", orgName)
+		assert.Equal(t, "fortify", repoName)
+	})
+	t.Run("valid repository 2", func(t *testing.T) {
+		repository := "https://github.hello.test/Testing/fortify"
+		toolInstance, orgName, repoName, err := parseRepositoryURL(repository)
+		assert.NoError(t, err)
+		assert.Equal(t, "https://github.hello.test", toolInstance)
+		assert.Equal(t, "Testing", orgName)
+		assert.Equal(t, "fortify", repoName)
+	})
+	t.Run("Invalid repository without repo name", func(t *testing.T) {
+		repository := "https://github.hello.test/Testing"
+		toolInstance, orgName, repoName, err := parseRepositoryURL(repository)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "Unable to parse organization and repo names")
+		assert.Equal(t, "", toolInstance)
+		assert.Equal(t, "", orgName)
+		assert.Equal(t, "", repoName)
+	})
+	t.Run("Invalid repository without organization name", func(t *testing.T) {
+		repository := "https://github.hello.test/fortify"
+		toolInstance, orgName, repoName, err := parseRepositoryURL(repository)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "Unable to parse organization and repo names")
+		assert.Equal(t, "", toolInstance)
+		assert.Equal(t, "", orgName)
+		assert.Equal(t, "", repoName)
+	})
+	t.Run("Invalid repository without tool instance", func(t *testing.T) {
+		repository := "/Testing/fortify"
+		toolInstance, orgName, repoName, err := parseRepositoryURL(repository)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "Unable to parse tool instance")
+		assert.Equal(t, "", toolInstance)
+		assert.Equal(t, "", orgName)
+		assert.Equal(t, "", repoName)
+	})
+	t.Run("Empty repository", func(t *testing.T) {
+		repository := ""
+		toolInstance, orgName, repoName, err := parseRepositoryURL(repository)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "Repository param is not set")
+		assert.Equal(t, "", toolInstance)
+		assert.Equal(t, "", orgName)
+		assert.Equal(t, "", repoName)
+	})
+}
+
+func TestBuildRepoReference(t *testing.T) {
+	t.Run("Valid ref with branch", func(t *testing.T) {
+		repository := "https://github.hello.test/Testing/fortify"
+		analyzedRef := "refs/head/branch"
+		ref, err := buildRepoReference(repository, analyzedRef)
+		assert.NoError(t, err)
+		assert.Equal(t, "https://github.hello.test/Testing/fortify/tree/branch", ref)
+	})
+	t.Run("Valid ref with PR", func(t *testing.T) {
+		repository := "https://github.hello.test/Testing/fortify"
+		analyzedRef := "refs/pull/1/merge"
+		ref, err := buildRepoReference(repository, analyzedRef)
+		assert.NoError(t, err)
+		assert.Equal(t, "https://github.hello.test/Testing/fortify/pull/1", ref)
+	})
+	t.Run("Invalid ref without branch name", func(t *testing.T) {
+		repository := "https://github.hello.test/Testing/fortify"
+		analyzedRef := "refs/head"
+		ref, err := buildRepoReference(repository, analyzedRef)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "Wrong analyzedRef format")
+		assert.Equal(t, "", ref)
+	})
+	t.Run("Invalid ref without PR id", func(t *testing.T) {
+		repository := "https://github.hello.test/Testing/fortify"
+		analyzedRef := "refs/pull/merge"
+		ref, err := buildRepoReference(repository, analyzedRef)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "Wrong analyzedRef format")
+		assert.Equal(t, "", ref)
+	})
+	t.Run("Empty repository", func(t *testing.T) {
+		repository := ""
+		analyzedRef := "refs/pull/merge"
+		ref, err := buildRepoReference(repository, analyzedRef)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "Repository or analyzedRef param is not set")
+		assert.Equal(t, "", ref)
+	})
+	t.Run("Empty analyzedRef", func(t *testing.T) {
+		repository := "https://github.hello.test/Testing/fortify"
+		analyzedRef := ""
+		ref, err := buildRepoReference(repository, analyzedRef)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "Repository or analyzedRef param is not set")
+		assert.Equal(t, "", ref)
+	})
+}
+
+func TestCreateToolRecordCodeql(t *testing.T) {
+	t.Run("Valid toolrun file", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{
+			Repository:  "https://github.hello.test/Testing/fortify.git",
+			AnalyzedRef: "refs/head/branch",
+			CommitID:    "test",
+		}
+		fileName, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), "test", config)
+		assert.NoError(t, err)
+		assert.Contains(t, fileName, "toolrun_codeql")
+	})
+	t.Run("Empty repository URL", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{
+			Repository:  "",
+			AnalyzedRef: "refs/head/branch",
+			CommitID:    "test",
+		}
+		fileName, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), "", config)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "Repository param is not set")
+		assert.Empty(t, fileName)
+	})
+	t.Run("Invalid repository URL", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{
+			Repository:  "https://github.hello.test/Testing",
+			AnalyzedRef: "refs/head/branch",
+			CommitID:    "test",
+		}
+		fileName, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), "test", config)
+		assert.Error(t, err)
+		assert.Regexp(t, "^Unable to parse [a-z ]+ from repository url$", err.Error())
+		assert.Empty(t, fileName)
+	})
+	t.Run("Empty workspace", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{
+			Repository:  "https://github.hello.test/Testing/fortify.git",
+			AnalyzedRef: "refs/head/branch",
+			CommitID:    "test",
+		}
+		fileName, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), "", config)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "TR_PERSIST: empty workspace")
+		assert.Empty(t, fileName)
+	})
+	t.Run("Empty analyzedRef", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{
+			Repository:  "https://github.hello.test/Testing/fortify.git",
+			AnalyzedRef: "",
+			CommitID:    "test",
+		}
+		fileName, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), "test", config)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "TR_ADD_KEY: empty keyvalue")
+		assert.Empty(t, fileName, "toolrun_codeql")
+	})
+	t.Run("Invalid analyzedRef", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{
+			Repository:  "https://github.hello.test/Testing/fortify.git",
+			AnalyzedRef: "refs/head",
+			CommitID:    "test",
+		}
+		fileName, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), "test", config)
+		assert.NoError(t, err)
+		assert.Contains(t, fileName, "toolrun_codeql")
+	})
+}

resources/metadata/codeqlExecuteScan.yaml

@@ -139,3 +139,5 @@ spec:
             type: codeql
           - filePattern: "**/*.sarif"
             type: codeql
+          - filePattern: "**/toolrun_codeql_*.json"
+            type: codeql