From ca633186f95a99f2128da06ecc291eadd5b60270 Mon Sep 17 00:00:00 2001 From: Daria Kuznetsova Date: Mon, 20 Nov 2023 14:21:04 +0100 Subject: [PATCH] refactor(codeqlExecuteScan): refactor codeql reporting (#4682) Co-authored-by: sumeet patil --- cmd/codeqlExecuteScan.go | 160 +++++----------------- cmd/codeqlExecuteScan_test.go | 246 +++++++++++----------------------- pkg/codeql/reporting.go | 92 +++++++++++++ pkg/codeql/reporting_test.go | 108 +++++++++++++++ 4 files changed, 315 insertions(+), 291 deletions(-) create mode 100644 pkg/codeql/reporting_test.go diff --git a/cmd/codeqlExecuteScan.go b/cmd/codeqlExecuteScan.go index 7b52c174f..fd8b1833a 100644 --- a/cmd/codeqlExecuteScan.go +++ b/cmd/codeqlExecuteScan.go @@ -15,7 +15,6 @@ import ( "github.com/SAP/jenkins-library/pkg/orchestrator" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/telemetry" - "github.com/SAP/jenkins-library/pkg/toolrecord" "github.com/pkg/errors" ) @@ -25,14 +24,6 @@ type codeqlExecuteScanUtils interface { piperutils.FileUtils } -type RepoInfo struct { - serverUrl string - repo string - commitId string - ref string - owner string -} - type codeqlExecuteScanUtilsBundle struct { *command.Command *piperutils.Files @@ -97,7 +88,7 @@ func getLangFromBuildTool(buildTool string) string { } } -func getGitRepoInfo(repoUri string, repoInfo *RepoInfo) error { +func getGitRepoInfo(repoUri string, repoInfo *codeql.RepoInfo) error { if repoUri == "" { return errors.New("repository param is not set or it cannot be auto populated") } @@ -106,43 +97,43 @@ func getGitRepoInfo(repoUri string, repoInfo *RepoInfo) error { matches := pat.FindAllStringSubmatch(repoUri, -1) if len(matches) > 0 { match := matches[0] - repoInfo.serverUrl = "https://" + match[3] + repoInfo.ServerUrl = "https://" + match[3] repoData := strings.Split(strings.TrimSuffix(match[4], ".git"), "/") if len(repoData) != 2 { return fmt.Errorf("Invalid repository %s", repoUri) } - repoInfo.owner = repoData[0] - repoInfo.repo = repoData[1] + repoInfo.Owner = repoData[0] + repoInfo.Repo = repoData[1] return nil } return fmt.Errorf("Invalid repository %s", repoUri) } -func initGitInfo(config *codeqlExecuteScanOptions) (RepoInfo, error) { - var repoInfo RepoInfo +func initGitInfo(config *codeqlExecuteScanOptions) (codeql.RepoInfo, error) { + var repoInfo codeql.RepoInfo err := getGitRepoInfo(config.Repository, &repoInfo) if err != nil { log.Entry().Error(err) } - repoInfo.ref = config.AnalyzedRef - repoInfo.commitId = config.CommitID + repoInfo.Ref = config.AnalyzedRef + repoInfo.CommitId = config.CommitID provider, err := orchestrator.NewOrchestratorSpecificConfigProvider() if err != nil { log.Entry().Warn("No orchestrator found. We assume piper is running locally.") } else { - if repoInfo.ref == "" { - repoInfo.ref = provider.GetReference() + if repoInfo.Ref == "" { + repoInfo.Ref = provider.GetReference() } - if repoInfo.commitId == "" || repoInfo.commitId == "NA" { - repoInfo.commitId = provider.GetCommit() + if repoInfo.CommitId == "" || repoInfo.CommitId == "NA" { + repoInfo.CommitId = provider.GetCommit() } - if repoInfo.serverUrl == "" { + if repoInfo.ServerUrl == "" { err = getGitRepoInfo(provider.GetRepoURL(), &repoInfo) if err != nil { log.Entry().Error(err) @@ -150,7 +141,7 @@ func initGitInfo(config *codeqlExecuteScanOptions) (RepoInfo, error) { } } if len(config.TargetGithubRepoURL) > 0 { - if strings.Contains(repoInfo.serverUrl, "github") { + if strings.Contains(repoInfo.ServerUrl, "github") { log.Entry().Errorf("TargetGithubRepoURL should not be set as the source repo is on github.") return repoInfo, errors.New("TargetGithubRepoURL should not be set as the source repo is on github.") } @@ -160,9 +151,9 @@ func initGitInfo(config *codeqlExecuteScanOptions) (RepoInfo, error) { return repoInfo, err } if len(config.TargetGithubBranchName) > 0 { - repoInfo.ref = config.TargetGithubBranchName + repoInfo.Ref = config.TargetGithubBranchName if len(strings.Split(config.TargetGithubBranchName, "/")) < 3 { - repoInfo.ref = "refs/heads/" + config.TargetGithubBranchName + repoInfo.Ref = "refs/heads/" + config.TargetGithubBranchName } } } @@ -183,27 +174,27 @@ func getToken(config *codeqlExecuteScanOptions) (bool, string) { return false, "" } -func uploadResults(config *codeqlExecuteScanOptions, repoInfo RepoInfo, token string, utils codeqlExecuteScanUtils) (string, error) { +func uploadResults(config *codeqlExecuteScanOptions, repoInfo codeql.RepoInfo, token string, utils codeqlExecuteScanUtils) (string, error) { cmd := []string{"github", "upload-results", "--sarif=" + filepath.Join(config.ModulePath, "target", "codeqlReport.sarif")} if config.GithubToken != "" { cmd = append(cmd, "-a="+token) } - if repoInfo.commitId != "" { - cmd = append(cmd, "--commit="+repoInfo.commitId) + if repoInfo.CommitId != "" { + cmd = append(cmd, "--commit="+repoInfo.CommitId) } - if repoInfo.serverUrl != "" { - cmd = append(cmd, "--github-url="+repoInfo.serverUrl) + if repoInfo.ServerUrl != "" { + cmd = append(cmd, "--github-url="+repoInfo.ServerUrl) } - if repoInfo.repo != "" { - cmd = append(cmd, "--repository="+(repoInfo.owner+"/"+repoInfo.repo)) + if repoInfo.Repo != "" { + cmd = append(cmd, "--repository="+(repoInfo.Owner+"/"+repoInfo.Repo)) } - if repoInfo.ref != "" { - cmd = append(cmd, "--ref="+repoInfo.ref) + if repoInfo.Ref != "" { + cmd = append(cmd, "--ref="+repoInfo.Ref) } //if no git params are passed(commitId, reference, serverUrl, repository), then codeql tries to auto populate it based on git information of the checkout repository. @@ -333,9 +324,9 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem if err != nil { return reports, err } - repoUrl := fmt.Sprintf("%s/%s/%s", repoInfo.serverUrl, repoInfo.owner, repoInfo.repo) - repoReference, err := buildRepoReference(repoUrl, repoInfo.ref) - repoCodeqlScanUrl := fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.ref) + repoUrl := fmt.Sprintf("%s/%s/%s", repoInfo.ServerUrl, repoInfo.Owner, repoInfo.Repo) + repoReference, err := codeql.BuildRepoReference(repoUrl, repoInfo.Ref) + repoCodeqlScanUrl := fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.Ref) if len(config.TargetGithubRepoURL) > 0 { hasToken, token := getToken(config) @@ -344,9 +335,9 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem } repoUploader, err := codeql.NewGitUploaderInstance( token, - repoInfo.ref, + repoInfo.Ref, config.Database, - repoInfo.commitId, + repoInfo.CommitId, config.Repository, config.TargetGithubRepoURL, ) @@ -357,7 +348,7 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem if err != nil { return reports, errors.Wrap(err, "failed uploading db sources from non-GitHub SCM to GitHub") } - repoInfo.commitId = targetCommitId + repoInfo.CommitId = targetCommitId } if !config.UploadResults { @@ -378,8 +369,8 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem return reports, errors.Wrap(err, "failed to upload sarif") } - codeqlScanAuditInstance := codeql.NewCodeqlScanAuditInstance(repoInfo.serverUrl, repoInfo.owner, repoInfo.repo, token, []string{}) - scanResults, err := codeqlScanAuditInstance.GetVulnerabilities(repoInfo.ref) + codeqlScanAuditInstance := codeql.NewCodeqlScanAuditInstance(repoInfo.ServerUrl, repoInfo.Owner, repoInfo.Repo, token, []string{}) + scanResults, err := codeqlScanAuditInstance.GetVulnerabilities(repoInfo.Ref) if err != nil { return reports, errors.Wrap(err, "failed to get scan results") } @@ -395,14 +386,14 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem for _, scanResult := range scanResults { unaudited := scanResult.Total - scanResult.Audited if unaudited > config.VulnerabilityThresholdTotal { - msg := fmt.Sprintf("Your repository %v with ref %v is not compliant. Total unaudited issues are %v which is greater than the VulnerabilityThresholdTotal count %v", repoUrl, repoInfo.ref, unaudited, config.VulnerabilityThresholdTotal) + msg := fmt.Sprintf("Your repository %v with ref %v is not compliant. Total unaudited issues are %v which is greater than the VulnerabilityThresholdTotal count %v", repoUrl, repoInfo.Ref, unaudited, config.VulnerabilityThresholdTotal) return reports, errors.Errorf(msg) } } } } - toolRecordFileName, err := createAndPersistToolRecord(utils, repoInfo, repoReference, repoUrl, repoCodeqlScanUrl) + toolRecordFileName, err := codeql.CreateAndPersistToolRecord(utils, repoInfo, repoReference, repoUrl, config.ModulePath) if err != nil { log.Entry().Warning("TR_CODEQL: Failed to create toolrecord file ...", err) } else { @@ -412,87 +403,6 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem return reports, nil } -func createAndPersistToolRecord(utils codeqlExecuteScanUtils, repoInfo RepoInfo, repoReference string, repoUrl string, repoCodeqlScanUrl string) (string, error) { - toolRecord, err := createToolRecordCodeql(utils, repoInfo, repoReference, repoUrl, repoCodeqlScanUrl) - if err != nil { - return "", err - } - - toolRecordFileName, err := persistToolRecord(toolRecord) - if err != nil { - return "", err - } - - return toolRecordFileName, nil -} - -func createToolRecordCodeql(utils codeqlExecuteScanUtils, repoInfo RepoInfo, repoUrl string, repoReference string, repoCodeqlScanUrl string) (*toolrecord.Toolrecord, error) { - record := toolrecord.New(utils, "./", "codeql", repoInfo.serverUrl) - - if repoInfo.serverUrl == "" { - return record, errors.New("Repository not set") - } - - if repoInfo.commitId == "" || repoInfo.commitId == "NA" { - return record, errors.New("CommitId not set") - } - - if repoInfo.ref == "" { - return record, errors.New("Analyzed Reference not set") - } - - record.DisplayName = fmt.Sprintf("%s %s - %s %s", repoInfo.owner, repoInfo.repo, repoInfo.ref, repoInfo.commitId) - record.DisplayURL = fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.ref) - - err := record.AddKeyData("repository", - fmt.Sprintf("%s/%s", repoInfo.owner, repoInfo.repo), - fmt.Sprintf("%s %s", repoInfo.owner, repoInfo.repo), - repoUrl) - if err != nil { - return record, err - } - - err = record.AddKeyData("repositoryReference", - repoInfo.ref, - fmt.Sprintf("%s - %s", repoInfo.repo, repoInfo.ref), - repoReference) - if err != nil { - return record, err - } - - err = record.AddKeyData("scanResult", - fmt.Sprintf("%s/%s", repoInfo.ref, repoInfo.commitId), - fmt.Sprintf("%s %s - %s %s", repoInfo.owner, repoInfo.repo, repoInfo.ref, repoInfo.commitId), - fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.ref)) - if err != nil { - return record, err - } - - return record, nil -} - -func buildRepoReference(repository, analyzedRef string) (string, error) { - ref := strings.Split(analyzedRef, "/") - if len(ref) < 3 { - return "", errors.New(fmt.Sprintf("Wrong analyzedRef format: %s", analyzedRef)) - } - if strings.Contains(analyzedRef, "pull") { - if len(ref) < 4 { - return "", errors.New(fmt.Sprintf("Wrong analyzedRef format: %s", analyzedRef)) - } - return fmt.Sprintf("%s/pull/%s", repository, ref[2]), nil - } - return fmt.Sprintf("%s/tree/%s", repository, ref[2]), nil -} - -func persistToolRecord(toolRecord *toolrecord.Toolrecord) (string, error) { - err := toolRecord.Persist() - if err != nil { - return "", err - } - return toolRecord.GetFileName(), nil -} - func getRamAndThreadsFromConfig(config *codeqlExecuteScanOptions) []string { params := make([]string, 0, 2) if len(config.Threads) > 0 { diff --git a/cmd/codeqlExecuteScan_test.go b/cmd/codeqlExecuteScan_test.go index ffcfc57e4..0a2d79d7e 100644 --- a/cmd/codeqlExecuteScan_test.go +++ b/cmd/codeqlExecuteScan_test.go @@ -4,7 +4,6 @@ package cmd import ( - "fmt" "testing" "time" @@ -75,104 +74,104 @@ func TestRunCodeqlExecuteScan(t *testing.T) { func TestGetGitRepoInfo(t *testing.T) { t.Run("Valid https URL1", func(t *testing.T) { - var repoInfo RepoInfo + var repoInfo codeql.RepoInfo err := getGitRepoInfo("https://github.hello.test/Testing/fortify.git", &repoInfo) assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) - assert.Equal(t, "fortify", repoInfo.repo) - assert.Equal(t, "Testing", repoInfo.owner) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) + assert.Equal(t, "fortify", repoInfo.Repo) + assert.Equal(t, "Testing", repoInfo.Owner) }) t.Run("Valid https URL2", func(t *testing.T) { - var repoInfo RepoInfo + var repoInfo codeql.RepoInfo err := getGitRepoInfo("https://github.hello.test/Testing/fortify", &repoInfo) assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) - assert.Equal(t, "fortify", repoInfo.repo) - assert.Equal(t, "Testing", repoInfo.owner) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) + assert.Equal(t, "fortify", repoInfo.Repo) + assert.Equal(t, "Testing", repoInfo.Owner) }) t.Run("Valid https URL1 with dots", func(t *testing.T) { - var repoInfo RepoInfo + var repoInfo codeql.RepoInfo err := getGitRepoInfo("https://github.hello.test/Testing/com.sap.fortify.git", &repoInfo) assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) - assert.Equal(t, "com.sap.fortify", repoInfo.repo) - assert.Equal(t, "Testing", repoInfo.owner) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) + assert.Equal(t, "com.sap.fortify", repoInfo.Repo) + assert.Equal(t, "Testing", repoInfo.Owner) }) t.Run("Valid https URL2 with dots", func(t *testing.T) { - var repoInfo RepoInfo + var repoInfo codeql.RepoInfo err := getGitRepoInfo("https://github.hello.test/Testing/com.sap.fortify", &repoInfo) assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) - assert.Equal(t, "com.sap.fortify", repoInfo.repo) - assert.Equal(t, "Testing", repoInfo.owner) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) + assert.Equal(t, "com.sap.fortify", repoInfo.Repo) + assert.Equal(t, "Testing", repoInfo.Owner) }) t.Run("Valid https URL1 with username and token", func(t *testing.T) { - var repoInfo RepoInfo + var repoInfo codeql.RepoInfo err := getGitRepoInfo("https://username:token@github.hello.test/Testing/fortify.git", &repoInfo) assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) - assert.Equal(t, "fortify", repoInfo.repo) - assert.Equal(t, "Testing", repoInfo.owner) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) + assert.Equal(t, "fortify", repoInfo.Repo) + assert.Equal(t, "Testing", repoInfo.Owner) }) t.Run("Valid https URL2 with username and token", func(t *testing.T) { - var repoInfo RepoInfo + var repoInfo codeql.RepoInfo err := getGitRepoInfo("https://username:token@github.hello.test/Testing/fortify", &repoInfo) assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) - assert.Equal(t, "fortify", repoInfo.repo) - assert.Equal(t, "Testing", repoInfo.owner) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) + assert.Equal(t, "fortify", repoInfo.Repo) + assert.Equal(t, "Testing", repoInfo.Owner) }) - t.Run("Invalid https URL as no org/owner passed", func(t *testing.T) { - var repoInfo RepoInfo + t.Run("Invalid https URL as no org/Owner passed", func(t *testing.T) { + var repoInfo codeql.RepoInfo assert.Error(t, getGitRepoInfo("https://github.com/fortify", &repoInfo)) }) t.Run("Invalid URL as no protocol passed", func(t *testing.T) { - var repoInfo RepoInfo + var repoInfo codeql.RepoInfo assert.Error(t, getGitRepoInfo("github.hello.test/Testing/fortify", &repoInfo)) }) t.Run("Valid ssh URL1", func(t *testing.T) { - var repoInfo RepoInfo + var repoInfo codeql.RepoInfo err := getGitRepoInfo("git@github.hello.test/Testing/fortify.git", &repoInfo) assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) - assert.Equal(t, "fortify", repoInfo.repo) - assert.Equal(t, "Testing", repoInfo.owner) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) + assert.Equal(t, "fortify", repoInfo.Repo) + assert.Equal(t, "Testing", repoInfo.Owner) }) t.Run("Valid ssh URL2", func(t *testing.T) { - var repoInfo RepoInfo + var repoInfo codeql.RepoInfo err := getGitRepoInfo("git@github.hello.test/Testing/fortify", &repoInfo) assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) - assert.Equal(t, "fortify", repoInfo.repo) - assert.Equal(t, "Testing", repoInfo.owner) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) + assert.Equal(t, "fortify", repoInfo.Repo) + assert.Equal(t, "Testing", repoInfo.Owner) }) t.Run("Valid ssh URL1 with dots", func(t *testing.T) { - var repoInfo RepoInfo + var repoInfo codeql.RepoInfo err := getGitRepoInfo("git@github.hello.test/Testing/com.sap.fortify.git", &repoInfo) assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) - assert.Equal(t, "com.sap.fortify", repoInfo.repo) - assert.Equal(t, "Testing", repoInfo.owner) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) + assert.Equal(t, "com.sap.fortify", repoInfo.Repo) + assert.Equal(t, "Testing", repoInfo.Owner) }) t.Run("Valid ssh URL2 with dots", func(t *testing.T) { - var repoInfo RepoInfo + var repoInfo codeql.RepoInfo err := getGitRepoInfo("git@github.hello.test/Testing/com.sap.fortify", &repoInfo) assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) - assert.Equal(t, "com.sap.fortify", repoInfo.repo) - assert.Equal(t, "Testing", repoInfo.owner) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) + assert.Equal(t, "com.sap.fortify", repoInfo.Repo) + assert.Equal(t, "Testing", repoInfo.Owner) }) - t.Run("Invalid ssh URL as no org/owner passed", func(t *testing.T) { - var repoInfo RepoInfo + t.Run("Invalid ssh URL as no org/Owner passed", func(t *testing.T) { + var repoInfo codeql.RepoInfo assert.Error(t, getGitRepoInfo("git@github.com/fortify", &repoInfo)) }) } @@ -182,66 +181,66 @@ func TestInitGitInfo(t *testing.T) { config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/codeql.git", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"} repoInfo, err := initGitInfo(&config) assert.NoError(t, err) - assert.Equal(t, "abcd1234", repoInfo.commitId) - assert.Equal(t, "Testing", repoInfo.owner) - assert.Equal(t, "codeql", repoInfo.repo) - assert.Equal(t, "refs/head/branch", repoInfo.ref) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) + assert.Equal(t, "abcd1234", repoInfo.CommitId) + assert.Equal(t, "Testing", repoInfo.Owner) + assert.Equal(t, "codeql", repoInfo.Repo) + assert.Equal(t, "refs/head/branch", repoInfo.Ref) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) }) t.Run("Valid URL2", func(t *testing.T) { config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/codeql", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"} repoInfo, err := initGitInfo(&config) assert.NoError(t, err) - assert.Equal(t, "abcd1234", repoInfo.commitId) - assert.Equal(t, "Testing", repoInfo.owner) - assert.Equal(t, "codeql", repoInfo.repo) - assert.Equal(t, "refs/head/branch", repoInfo.ref) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) + assert.Equal(t, "abcd1234", repoInfo.CommitId) + assert.Equal(t, "Testing", repoInfo.Owner) + assert.Equal(t, "codeql", repoInfo.Repo) + assert.Equal(t, "refs/head/branch", repoInfo.Ref) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) }) t.Run("Valid url with dots URL1", func(t *testing.T) { config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/com.sap.codeql.git", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"} repoInfo, err := initGitInfo(&config) assert.NoError(t, err) - assert.Equal(t, "abcd1234", repoInfo.commitId) - assert.Equal(t, "Testing", repoInfo.owner) - assert.Equal(t, "com.sap.codeql", repoInfo.repo) - assert.Equal(t, "refs/head/branch", repoInfo.ref) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) + assert.Equal(t, "abcd1234", repoInfo.CommitId) + assert.Equal(t, "Testing", repoInfo.Owner) + assert.Equal(t, "com.sap.codeql", repoInfo.Repo) + assert.Equal(t, "refs/head/branch", repoInfo.Ref) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) }) t.Run("Valid url with dots URL2", func(t *testing.T) { config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/com.sap.codeql", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"} repoInfo, err := initGitInfo(&config) assert.NoError(t, err) - assert.Equal(t, "abcd1234", repoInfo.commitId) - assert.Equal(t, "Testing", repoInfo.owner) - assert.Equal(t, "com.sap.codeql", repoInfo.repo) - assert.Equal(t, "refs/head/branch", repoInfo.ref) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) + assert.Equal(t, "abcd1234", repoInfo.CommitId) + assert.Equal(t, "Testing", repoInfo.Owner) + assert.Equal(t, "com.sap.codeql", repoInfo.Repo) + assert.Equal(t, "refs/head/branch", repoInfo.Ref) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) }) t.Run("Valid url with username and token URL1", func(t *testing.T) { config := codeqlExecuteScanOptions{Repository: "https://username:token@github.hello.test/Testing/codeql.git", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"} repoInfo, err := initGitInfo(&config) assert.NoError(t, err) - assert.Equal(t, "abcd1234", repoInfo.commitId) - assert.Equal(t, "Testing", repoInfo.owner) - assert.Equal(t, "codeql", repoInfo.repo) - assert.Equal(t, "refs/head/branch", repoInfo.ref) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) + assert.Equal(t, "abcd1234", repoInfo.CommitId) + assert.Equal(t, "Testing", repoInfo.Owner) + assert.Equal(t, "codeql", repoInfo.Repo) + assert.Equal(t, "refs/head/branch", repoInfo.Ref) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) }) t.Run("Valid url with username and token URL2", func(t *testing.T) { config := codeqlExecuteScanOptions{Repository: "https://username:token@github.hello.test/Testing/codeql", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"} repoInfo, err := initGitInfo(&config) assert.NoError(t, err) - assert.Equal(t, "abcd1234", repoInfo.commitId) - assert.Equal(t, "Testing", repoInfo.owner) - assert.Equal(t, "codeql", repoInfo.repo) - assert.Equal(t, "refs/head/branch", repoInfo.ref) - assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl) + assert.Equal(t, "abcd1234", repoInfo.CommitId) + assert.Equal(t, "Testing", repoInfo.Owner) + assert.Equal(t, "codeql", repoInfo.Repo) + assert.Equal(t, "refs/head/branch", repoInfo.Ref) + assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl) }) t.Run("Invalid URL with no org/reponame", func(t *testing.T) { @@ -249,101 +248,16 @@ func TestInitGitInfo(t *testing.T) { repoInfo, err := initGitInfo(&config) assert.NoError(t, err) _, err = orchestrator.NewOrchestratorSpecificConfigProvider() - assert.Equal(t, "abcd1234", repoInfo.commitId) - assert.Equal(t, "refs/head/branch", repoInfo.ref) + assert.Equal(t, "abcd1234", repoInfo.CommitId) + assert.Equal(t, "refs/head/branch", repoInfo.Ref) if err != nil { - assert.Equal(t, "", repoInfo.owner) - assert.Equal(t, "", repoInfo.repo) - assert.Equal(t, "", repoInfo.serverUrl) + assert.Equal(t, "", repoInfo.Owner) + assert.Equal(t, "", repoInfo.Repo) + assert.Equal(t, "", repoInfo.ServerUrl) } }) } -func TestBuildRepoReference(t *testing.T) { - t.Run("Valid ref with branch", func(t *testing.T) { - repository := "https://github.hello.test/Testing/fortify" - analyzedRef := "refs/head/branch" - ref, err := buildRepoReference(repository, analyzedRef) - assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test/Testing/fortify/tree/branch", ref) - }) - t.Run("Valid ref with PR", func(t *testing.T) { - repository := "https://github.hello.test/Testing/fortify" - analyzedRef := "refs/pull/1/merge" - ref, err := buildRepoReference(repository, analyzedRef) - assert.NoError(t, err) - assert.Equal(t, "https://github.hello.test/Testing/fortify/pull/1", ref) - }) - t.Run("Invalid ref without branch name", func(t *testing.T) { - repository := "https://github.hello.test/Testing/fortify" - analyzedRef := "refs/head" - ref, err := buildRepoReference(repository, analyzedRef) - assert.Error(t, err) - assert.ErrorContains(t, err, "Wrong analyzedRef format") - assert.Equal(t, "", ref) - }) - t.Run("Invalid ref without PR id", func(t *testing.T) { - repository := "https://github.hello.test/Testing/fortify" - analyzedRef := "refs/pull/merge" - ref, err := buildRepoReference(repository, analyzedRef) - assert.Error(t, err) - assert.ErrorContains(t, err, "Wrong analyzedRef format") - assert.Equal(t, "", ref) - }) -} - -func getRepoReferences(repoInfo RepoInfo) (string, string, string) { - repoUrl := fmt.Sprintf("%s/%s/%s", repoInfo.serverUrl, repoInfo.owner, repoInfo.repo) - repoReference, _ := buildRepoReference(repoUrl, repoInfo.ref) - repoCodeqlScanUrl := fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.ref) - return repoUrl, repoReference, repoCodeqlScanUrl -} -func TestCreateToolRecordCodeql(t *testing.T) { - t.Run("Valid toolrun file", func(t *testing.T) { - repoInfo := RepoInfo{serverUrl: "https://github.hello.test", commitId: "test", ref: "refs/head/branch", owner: "Testing", repo: "fortify"} - repoUrl, repoReference, repoCodeqlScanUrl := getRepoReferences(repoInfo) - toolRecord, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, repoCodeqlScanUrl) - assert.NoError(t, err) - assert.Equal(t, toolRecord.ToolName, "codeql") - assert.Equal(t, toolRecord.ToolInstance, "https://github.hello.test") - assert.Equal(t, toolRecord.DisplayName, "Testing fortify - refs/head/branch test") - assert.Equal(t, toolRecord.DisplayURL, "https://github.hello.test/Testing/fortify/security/code-scanning?query=is:open+ref:refs/head/branch") - }) - t.Run("Empty repository URL", func(t *testing.T) { - repoInfo := RepoInfo{serverUrl: "", commitId: "test", ref: "refs/head/branch", owner: "Testing", repo: "fortify"} - repoUrl, repoReference, repoCodeqlScanUrl := getRepoReferences(repoInfo) - _, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, repoCodeqlScanUrl) - - assert.Error(t, err) - assert.ErrorContains(t, err, "Repository not set") - }) - - t.Run("Empty analyzedRef", func(t *testing.T) { - repoInfo := RepoInfo{serverUrl: "https://github.hello.test", commitId: "test", ref: "", owner: "Testing", repo: "fortify"} - repoUrl, repoReference, repoCodeqlScanUrl := getRepoReferences(repoInfo) - _, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, repoCodeqlScanUrl) - - assert.Error(t, err) - assert.ErrorContains(t, err, "Analyzed Reference not set") - }) - - t.Run("Empty CommitId", func(t *testing.T) { - repoInfo := RepoInfo{serverUrl: "https://github.hello.test", commitId: "", ref: "refs/head/branch", owner: "Testing", repo: "fortify"} - repoUrl, repoReference, repoCodeqlScanUrl := getRepoReferences(repoInfo) - _, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, repoCodeqlScanUrl) - - assert.Error(t, err) - assert.ErrorContains(t, err, "CommitId not set") - }) - t.Run("Invalid analyzedRef", func(t *testing.T) { - repoInfo := RepoInfo{serverUrl: "https://github.hello.test", commitId: "", ref: "refs/branch", owner: "Testing", repo: "fortify"} - repoUrl, repoReference, repoCodeqlScanUrl := getRepoReferences(repoInfo) - _, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, repoCodeqlScanUrl) - - assert.Error(t, err) - }) -} - func TestWaitSarifUploaded(t *testing.T) { t.Parallel() config := codeqlExecuteScanOptions{SarifCheckRetryInterval: 1, SarifCheckMaxRetries: 5} diff --git a/pkg/codeql/reporting.go b/pkg/codeql/reporting.go index 380307d1e..fc095ccff 100644 --- a/pkg/codeql/reporting.go +++ b/pkg/codeql/reporting.go @@ -2,10 +2,13 @@ package codeql import ( "encoding/json" + "fmt" "path/filepath" + "strings" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" + "github.com/SAP/jenkins-library/pkg/toolrecord" "github.com/pkg/errors" ) @@ -24,6 +27,14 @@ type CodeqlFindings struct { Audited int `json:"audited"` } +type RepoInfo struct { + ServerUrl string + Repo string + CommitId string + Ref string + Owner string +} + func WriteJSONReport(jsonReport CodeqlAudit, modulePath string) ([]piperutils.Path, error) { utils := piperutils.Files{} reportPaths := []piperutils.Path{} @@ -44,3 +55,84 @@ func WriteJSONReport(jsonReport CodeqlAudit, modulePath string) ([]piperutils.Pa return reportPaths, nil } + +func BuildRepoReference(repository, analyzedRef string) (string, error) { + ref := strings.Split(analyzedRef, "/") + if len(ref) < 3 { + return "", errors.New(fmt.Sprintf("Wrong analyzedRef format: %s", analyzedRef)) + } + if strings.Contains(analyzedRef, "pull") { + if len(ref) < 4 { + return "", errors.New(fmt.Sprintf("Wrong analyzedRef format: %s", analyzedRef)) + } + return fmt.Sprintf("%s/pull/%s", repository, ref[2]), nil + } + return fmt.Sprintf("%s/tree/%s", repository, ref[2]), nil +} + +func CreateAndPersistToolRecord(utils piperutils.FileUtils, repoInfo RepoInfo, repoReference, repoUrl, modulePath string) (string, error) { + toolRecord, err := createToolRecordCodeql(utils, repoInfo, repoReference, repoUrl, modulePath) + if err != nil { + return "", err + } + + toolRecordFileName, err := persistToolRecord(toolRecord) + if err != nil { + return "", err + } + + return toolRecordFileName, nil +} + +func createToolRecordCodeql(utils piperutils.FileUtils, repoInfo RepoInfo, repoUrl, repoReference, modulePath string) (*toolrecord.Toolrecord, error) { + record := toolrecord.New(utils, modulePath, "codeql", repoInfo.ServerUrl) + + if repoInfo.ServerUrl == "" { + return record, errors.New("Repository not set") + } + + if repoInfo.CommitId == "" || repoInfo.CommitId == "NA" { + return record, errors.New("CommitId not set") + } + + if repoInfo.Ref == "" { + return record, errors.New("Analyzed Reference not set") + } + + record.DisplayName = fmt.Sprintf("%s %s - %s %s", repoInfo.Owner, repoInfo.Repo, repoInfo.Ref, repoInfo.CommitId) + record.DisplayURL = fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.Ref) + + err := record.AddKeyData("repository", + fmt.Sprintf("%s/%s", repoInfo.Owner, repoInfo.Repo), + fmt.Sprintf("%s %s", repoInfo.Owner, repoInfo.Repo), + repoUrl) + if err != nil { + return record, err + } + + err = record.AddKeyData("repositoryReference", + repoInfo.Ref, + fmt.Sprintf("%s - %s", repoInfo.Repo, repoInfo.Ref), + repoReference) + if err != nil { + return record, err + } + + err = record.AddKeyData("scanResult", + fmt.Sprintf("%s/%s", repoInfo.Ref, repoInfo.CommitId), + fmt.Sprintf("%s %s - %s %s", repoInfo.Owner, repoInfo.Repo, repoInfo.Ref, repoInfo.CommitId), + fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.Ref)) + if err != nil { + return record, err + } + + return record, nil +} + +func persistToolRecord(toolRecord *toolrecord.Toolrecord) (string, error) { + err := toolRecord.Persist() + if err != nil { + return "", err + } + return toolRecord.GetFileName(), nil +} diff --git a/pkg/codeql/reporting_test.go b/pkg/codeql/reporting_test.go new file mode 100644 index 000000000..2587a559f --- /dev/null +++ b/pkg/codeql/reporting_test.go @@ -0,0 +1,108 @@ +package codeql + +import ( + "fmt" + "testing" + + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/stretchr/testify/assert" +) + +type codeqlExecuteScanMockUtils struct { + *mock.ExecMockRunner + *mock.FilesMock +} + +func newCodeqlExecuteScanTestsUtils() codeqlExecuteScanMockUtils { + utils := codeqlExecuteScanMockUtils{ + ExecMockRunner: &mock.ExecMockRunner{}, + FilesMock: &mock.FilesMock{}, + } + return utils +} + +func TestBuildRepoReference(t *testing.T) { + t.Run("Valid Ref with branch", func(t *testing.T) { + repository := "https://github.hello.test/Testing/fortify" + analyzedRef := "refs/head/branch" + ref, err := BuildRepoReference(repository, analyzedRef) + assert.NoError(t, err) + assert.Equal(t, "https://github.hello.test/Testing/fortify/tree/branch", ref) + }) + t.Run("Valid Ref with PR", func(t *testing.T) { + repository := "https://github.hello.test/Testing/fortify" + analyzedRef := "refs/pull/1/merge" + ref, err := BuildRepoReference(repository, analyzedRef) + assert.NoError(t, err) + assert.Equal(t, "https://github.hello.test/Testing/fortify/pull/1", ref) + }) + t.Run("Invalid Ref without branch name", func(t *testing.T) { + repository := "https://github.hello.test/Testing/fortify" + analyzedRef := "refs/head" + ref, err := BuildRepoReference(repository, analyzedRef) + assert.Error(t, err) + assert.ErrorContains(t, err, "Wrong analyzedRef format") + assert.Equal(t, "", ref) + }) + t.Run("Invalid Ref without PR id", func(t *testing.T) { + repository := "https://github.hello.test/Testing/fortify" + analyzedRef := "refs/pull/merge" + ref, err := BuildRepoReference(repository, analyzedRef) + assert.Error(t, err) + assert.ErrorContains(t, err, "Wrong analyzedRef format") + assert.Equal(t, "", ref) + }) +} + +func getRepoReferences(repoInfo RepoInfo) (string, string) { + repoUrl := fmt.Sprintf("%s/%s/%s", repoInfo.ServerUrl, repoInfo.Owner, repoInfo.Repo) + repoReference, _ := BuildRepoReference(repoUrl, repoInfo.Ref) + return repoUrl, repoReference +} + +func TestCreateToolRecordCodeql(t *testing.T) { + modulePath := "./" + t.Run("Valid toolrun file", func(t *testing.T) { + repoInfo := RepoInfo{ServerUrl: "https://github.hello.test", CommitId: "test", Ref: "refs/head/branch", Owner: "Testing", Repo: "fortify"} + repoUrl, repoReference := getRepoReferences(repoInfo) + toolRecord, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, modulePath) + assert.NoError(t, err) + assert.Equal(t, toolRecord.ToolName, "codeql") + assert.Equal(t, toolRecord.ToolInstance, "https://github.hello.test") + assert.Equal(t, toolRecord.DisplayName, "Testing fortify - refs/head/branch test") + assert.Equal(t, toolRecord.DisplayURL, "https://github.hello.test/Testing/fortify/security/code-scanning?query=is:open+ref:refs/head/branch") + }) + t.Run("Empty repository URL", func(t *testing.T) { + repoInfo := RepoInfo{ServerUrl: "", CommitId: "test", Ref: "refs/head/branch", Owner: "Testing", Repo: "fortify"} + repoUrl, repoReference := getRepoReferences(repoInfo) + _, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, modulePath) + + assert.Error(t, err) + assert.ErrorContains(t, err, "Repository not set") + }) + + t.Run("Empty analyzedRef", func(t *testing.T) { + repoInfo := RepoInfo{ServerUrl: "https://github.hello.test", CommitId: "test", Ref: "", Owner: "Testing", Repo: "fortify"} + repoUrl, repoReference := getRepoReferences(repoInfo) + _, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, modulePath) + + assert.Error(t, err) + assert.ErrorContains(t, err, "Analyzed Reference not set") + }) + + t.Run("Empty CommitId", func(t *testing.T) { + repoInfo := RepoInfo{ServerUrl: "https://github.hello.test", CommitId: "", Ref: "refs/head/branch", Owner: "Testing", Repo: "fortify"} + repoUrl, repoReference := getRepoReferences(repoInfo) + _, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, modulePath) + + assert.Error(t, err) + assert.ErrorContains(t, err, "CommitId not set") + }) + t.Run("Invalid analyzedRef", func(t *testing.T) { + repoInfo := RepoInfo{ServerUrl: "https://github.hello.test", CommitId: "", Ref: "refs/branch", Owner: "Testing", Repo: "fortify"} + repoUrl, repoReference := getRepoReferences(repoInfo) + _, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, modulePath) + + assert.Error(t, err) + }) +}