jenkins/sap-jenkins-library
1
0
Fork 0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-03-27 21:49:15 +02:00
Code Issues Releases Activity

refactor codeqlExecuteScan (#4888)

Browse Source 
* refactored codeql step

* fixed tests

* fixed reports output dir

* refactored

* fixed returning error

* fixed tests

---------

Co-authored-by: sumeet patil <sumeet.patil@sap.com>
This commit is contained in:
Daria Kuznetsova 2024-04-16 10:35:56 +02:00 committed by GitHub
parent 94a33844a0
commit df0b288ecb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 976 additions and 650 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

393
cmd/codeqlExecuteScan.go
View File

@ -6,16 +6,13 @@ import (
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/SAP/jenkins-library/pkg/codeql"
"github.com/SAP/jenkins-library/pkg/command"
piperhttp "github.com/SAP/jenkins-library/pkg/http"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/maven"
"github.com/SAP/jenkins-library/pkg/orchestrator"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/SAP/jenkins-library/pkg/telemetry"
"github.com/pkg/errors"
@ -35,11 +32,6 @@ type codeqlExecuteScanUtilsBundle struct {
*piperhttp.Client
}
const (
sarifUploadComplete = "complete"
sarifUploadFailed = "failed"
)
func newCodeqlExecuteScanUtils() codeqlExecuteScanUtils {
utils := codeqlExecuteScanUtilsBundle{
Command: &command.Command{},
@ -53,7 +45,6 @@ func newCodeqlExecuteScanUtils() codeqlExecuteScanUtils {
}
func codeqlExecuteScan(config codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, influx *codeqlExecuteScanInflux) {
utils := newCodeqlExecuteScanUtils()
influx.step_data.fields.codeql = false
@ -100,81 +91,6 @@ func getLangFromBuildTool(buildTool string) string {
}
}
func getGitRepoInfo(repoUri string, repoInfo *codeql.RepoInfo) error {
if repoUri == "" {
return errors.New("repository param is not set or it cannot be auto populated")
}
pat := regexp.MustCompile(`^(https:\/\/|git@)([\S]+:[\S]+@)?([^\/:]+)[\/:]([^\/:]+\/[\S]+)$`)
matches := pat.FindAllStringSubmatch(repoUri, -1)
if len(matches) > 0 {
match := matches[0]
repoInfo.ServerUrl = "https://" + match[3]
repoData := strings.Split(strings.TrimSuffix(match[4], ".git"), "/")
if len(repoData) != 2 {
return fmt.Errorf("Invalid repository %s", repoUri)
}
repoInfo.Owner = repoData[0]
repoInfo.Repo = repoData[1]
return nil
}
return fmt.Errorf("Invalid repository %s", repoUri)
}
func initGitInfo(config *codeqlExecuteScanOptions) (codeql.RepoInfo, error) {
var repoInfo codeql.RepoInfo
err := getGitRepoInfo(config.Repository, &repoInfo)
if err != nil {
log.Entry().Error(err)
}
repoInfo.Ref = config.AnalyzedRef
repoInfo.CommitId = config.CommitID
provider, err := orchestrator.GetOrchestratorConfigProvider(nil)
if err != nil {
log.Entry().Warn("No orchestrator found. We assume piper is running locally.")
} else {
if repoInfo.Ref == "" {
repoInfo.Ref = provider.GitReference()
}
if repoInfo.CommitId == "" || repoInfo.CommitId == "NA" {
repoInfo.CommitId = provider.CommitSHA()
}
if repoInfo.ServerUrl == "" {
err = getGitRepoInfo(provider.RepoURL(), &repoInfo)
if err != nil {
log.Entry().Error(err)
}
}
}
if len(config.TargetGithubRepoURL) > 0 {
log.Entry().Infof("Checking target GitHub repo URL: %s", config.TargetGithubRepoURL)
if strings.Contains(repoInfo.ServerUrl, "github") {
log.Entry().Errorf("TargetGithubRepoURL should not be set as the source repo is on github.")
return repoInfo, errors.New("TargetGithubRepoURL should not be set as the source repo is on github.")
}
err := getGitRepoInfo(config.TargetGithubRepoURL, &repoInfo)
if err != nil {
log.Entry().Error(err)
return repoInfo, err
}
if len(config.TargetGithubBranchName) > 0 {
log.Entry().Infof("Target GitHub branch name: %s", config.TargetGithubBranchName)
repoInfo.Ref = config.TargetGithubBranchName
if len(strings.Split(config.TargetGithubBranchName, "/")) < 3 {
repoInfo.Ref = "refs/heads/" + config.TargetGithubBranchName
}
}
}
return repoInfo, nil
}
func getToken(config *codeqlExecuteScanOptions) (bool, string) {
if len(config.GithubToken) > 0 {
return true, config.GithubToken
@ -188,200 +104,106 @@ func getToken(config *codeqlExecuteScanOptions) (bool, string) {
return false, ""
}
func uploadResults(config *codeqlExecuteScanOptions, repoInfo codeql.RepoInfo, token string, utils codeqlExecuteScanUtils) (string, error) {
cmd := prepareCmdForUploadResults(config, &repoInfo, token)
//if no git params are passed(commitId, reference, serverUrl, repository), then codeql tries to auto populate it based on git information of the checkout repository.
//It also depends on the orchestrator. Some orchestrator keep git information and some not.
var bufferOut, bufferErr bytes.Buffer
utils.Stdout(&bufferOut)
defer utils.Stdout(log.Writer())
utils.Stderr(&bufferErr)
defer utils.Stderr(log.Writer())
err := execute(utils, cmd, GeneralConfig.Verbose)
if err != nil {
e := bufferErr.String()
log.Entry().Error(e)
if strings.Contains(e, "Unauthorized") {
log.Entry().Error("Either your Github Token is invalid or you use both Vault and Jenkins credentials where your Vault credentials are invalid, to use your Jenkins credentials try setting 'skipVault:true'")
}
log.Entry().Error("failed to upload sarif results")
return "", err
}
url := bufferOut.String()
return strings.TrimSpace(url), nil
}
func waitSarifUploaded(config *codeqlExecuteScanOptions, codeqlSarifUploader codeql.CodeqlSarifUploader) error {
maxRetries := config.SarifCheckMaxRetries
retryInterval := time.Duration(config.SarifCheckRetryInterval) * time.Second
log.Entry().Info("waiting for the SARIF to upload")
i := 1
for {
sarifStatus, err := codeqlSarifUploader.GetSarifStatus()
if err != nil {
return err
}
log.Entry().Infof("the SARIF processing status: %s", sarifStatus.ProcessingStatus)
if sarifStatus.ProcessingStatus == sarifUploadComplete {
return nil
}
if sarifStatus.ProcessingStatus == sarifUploadFailed {
for e := range sarifStatus.Errors {
log.Entry().Error(e)
}
return errors.New("failed to upload sarif file")
}
if i <= maxRetries {
log.Entry().Infof("still waiting for the SARIF to upload: retrying in %d seconds... (retry %d/%d)", config.SarifCheckRetryInterval, i, maxRetries)
time.Sleep(retryInterval)
i++
continue
}
return errors.New("failed to check sarif uploading status: max retries reached")
}
}
func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, utils codeqlExecuteScanUtils, influx *codeqlExecuteScanInflux) ([]piperutils.Path, error) {
func printCodeqlImageVersion() {
codeqlVersion, err := os.ReadFile("/etc/image-version")
if err != nil {
log.Entry().Infof("CodeQL image version: unknown")
} else {
log.Entry().Infof("CodeQL image version: %s", string(codeqlVersion))
}
}
func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, utils codeqlExecuteScanUtils, influx *codeqlExecuteScanInflux) ([]piperutils.Path, error) {
printCodeqlImageVersion()
var reports []piperutils.Path
dbCreateCustomFlags := codeql.ParseCustomFlags(config.DatabaseCreateFlags)
cmd, err := prepareCmdForDatabaseCreate(dbCreateCustomFlags, config, utils)
err := runDatabaseCreate(config, dbCreateCustomFlags, utils)
if err != nil {
log.Entry().WithError(err).Error("failed to prepare command for codeql database create")
return reports, err
}
err = execute(utils, cmd, GeneralConfig.Verbose)
if err != nil {
log.Entry().Error("failed running command codeql database create")
log.Entry().WithError(err).Error("failed to create codeql database")
return reports, err
}
err = os.MkdirAll(filepath.Join(config.ModulePath, "target"), os.ModePerm)
if err != nil {
return reports, fmt.Errorf("failed to create directory: %w", err)
log.Entry().WithError(err).Error("failed to create output directory for reports")
return reports, err
}
dbAnalyzeCustomFlags := codeql.ParseCustomFlags(config.DatabaseAnalyzeFlags)
cmd, err = prepareCmdForDatabaseAnalyze(dbAnalyzeCustomFlags, config, "sarif-latest", "codeqlReport.sarif")
scanReports, err := runDatabaseAnalyze(config, dbAnalyzeCustomFlags, utils)
if err != nil {
log.Entry().WithError(err).Error("failed to prepare command for codeql database analyze format=sarif-latest")
log.Entry().WithError(err).Error("failed to analyze codeql database")
return reports, err
}
err = execute(utils, cmd, GeneralConfig.Verbose)
if err != nil {
log.Entry().Error("failed running command codeql database analyze for sarif generation")
return reports, err
}
reports = append(reports, piperutils.Path{Target: filepath.Join(config.ModulePath, "target", "codeqlReport.sarif")})
reports = append(reports, scanReports...)
cmd, err = prepareCmdForDatabaseAnalyze(dbAnalyzeCustomFlags, config, "csv", "codeqlReport.csv")
repoInfo, err := codeql.GetRepoInfo(config.Repository, config.AnalyzedRef, config.CommitID,
config.TargetGithubRepoURL, config.TargetGithubBranchName)
if err != nil {
log.Entry().WithError(err).Error("failed to prepare command for codeql database analyze format=csv")
log.Entry().WithError(err).Error("failed to get repository info")
return reports, err
}
err = execute(utils, cmd, GeneralConfig.Verbose)
if err != nil {
log.Entry().Error("failed running command codeql database analyze for csv generation")
return reports, err
}
reports = append(reports, piperutils.Path{Target: filepath.Join(config.ModulePath, "target", "codeqlReport.csv")})
repoInfo, err := initGitInfo(config)
if err != nil {
return reports, err
}
repoUrl := fmt.Sprintf("%s/%s/%s", repoInfo.ServerUrl, repoInfo.Owner, repoInfo.Repo)
repoReference, err := codeql.BuildRepoReference(repoUrl, repoInfo.Ref)
repoCodeqlScanUrl := fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.Ref)
if len(config.TargetGithubRepoURL) > 0 {
log.Entry().Infof("DB sources for %s will be uploaded to target GitHub repo: %s", config.Repository, repoUrl)
hasToken, token := getToken(config)
if !hasToken {
return reports, errors.New("failed running upload db sources to GitHub as githubToken was not specified")
}
repoUploader, err := codeql.NewGitUploaderInstance(
token,
repoInfo.Ref,
config.Database,
repoInfo.CommitId,
config.Repository,
config.TargetGithubRepoURL,
)
err = uploadProjectToGitHub(config, repoInfo)
if err != nil {
log.Entry().WithError(err).Error("failed to upload project to Github")
return reports, err
}
targetCommitId, err := repoUploader.UploadProjectToGithub()
if err != nil {
return reports, errors.Wrap(err, "failed uploading db sources from non-GitHub SCM to GitHub")
}
repoInfo.CommitId = targetCommitId
log.Entry().Info("DB sources were successfully uploaded to target GitHub repo")
}
var scanResults []codeql.CodeqlFindings
if !config.UploadResults {
log.Entry().Warn("The sarif results will not be uploaded to the repository and compliance report will not be generated as uploadResults is set to false.")
} else {
log.Entry().Infof("The sarif results will be uploaded to the repository %s", repoUrl)
log.Entry().Infof("The sarif results will be uploaded to the repository %s", repoInfo.FullUrl)
hasToken, token := getToken(config)
if !hasToken {
return reports, errors.New("failed running upload-results as githubToken was not specified")
return reports, fmt.Errorf("failed running upload-results as githubToken was not specified")
}
sarifUrl, err := uploadResults(config, repoInfo, token, utils)
err = uploadSarifResults(config, token, repoInfo, utils)
if err != nil {
log.Entry().WithError(err).Error("failed to upload sarif results")
return reports, err
}
codeqlSarifUploader := codeql.NewCodeqlSarifUploaderInstance(sarifUrl, token)
err = waitSarifUploaded(config, &codeqlSarifUploader)
if err != nil {
return reports, errors.Wrap(err, "failed to upload sarif")
}
codeqlScanAuditInstance := codeql.NewCodeqlScanAuditInstance(repoInfo.ServerUrl, repoInfo.Owner, repoInfo.Repo, token, []string{})
scanResults, err = codeqlScanAuditInstance.GetVulnerabilities(repoInfo.Ref)
scanResults, err = codeqlScanAuditInstance.GetVulnerabilities(repoInfo.AnalyzedRef)
if err != nil {
return reports, errors.Wrap(err, "failed to get scan results")
log.Entry().WithError(err).Error("failed to get vulnerabilities")
return reports, err
}
codeqlAudit := codeql.CodeqlAudit{ToolName: "codeql", RepositoryUrl: repoUrl, CodeScanningLink: repoCodeqlScanUrl, RepositoryReferenceUrl: repoReference, QuerySuite: config.QuerySuite, ScanResults: scanResults}
codeqlAudit := codeql.CodeqlAudit{
ToolName: "codeql",
RepositoryUrl: repoInfo.FullUrl,
CodeScanningLink: repoInfo.ScanUrl,
RepositoryReferenceUrl: repoInfo.FullRef,
QuerySuite: config.QuerySuite,
ScanResults: scanResults,
}
paths, err := codeql.WriteJSONReport(codeqlAudit, config.ModulePath)
if err != nil {
return reports, errors.Wrap(err, "failed to write json compliance report")
log.Entry().WithError(err).Error("failed to write json compliance report")
return reports, err
}
reports = append(reports, paths...)
if config.CheckForCompliance {
for _, scanResult := range scanResults {
if scanResult.ClassificationName == codeql.AuditAll {
unaudited := scanResult.Total - scanResult.Audited
if unaudited > config.VulnerabilityThresholdTotal {
msg := fmt.Sprintf("Your repository %v with ref %v is not compliant. Total unaudited issues are %v which is greater than the VulnerabilityThresholdTotal count %v", repoUrl, repoInfo.Ref, unaudited, config.VulnerabilityThresholdTotal)
return reports, errors.Errorf(msg)
}
}
err = checkForCompliance(scanResults, config, repoInfo)
if err != nil {
return reports, err
}
}
}
addDataToInfluxDB(repoUrl, repoReference, repoCodeqlScanUrl, config.QuerySuite, scanResults, influx)
addDataToInfluxDB(repoInfo, config.QuerySuite, scanResults, influx)
toolRecordFileName, err := codeql.CreateAndPersistToolRecord(utils, repoInfo, repoReference, repoUrl, config.ModulePath)
toolRecordFileName, err := codeql.CreateAndPersistToolRecord(utils, repoInfo, config.ModulePath)
if err != nil {
log.Entry().Warning("TR_CODEQL: Failed to create toolrecord file ...", err)
} else {
@ -391,6 +213,70 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem
return reports, nil
}
func runDatabaseCreate(config *codeqlExecuteScanOptions, customFlags map[string]string, utils codeqlExecuteScanUtils) error {
cmd, err := prepareCmdForDatabaseCreate(customFlags, config, utils)
if err != nil {
log.Entry().Error("failed to prepare command for codeql database create")
return err
}
if err = execute(utils, cmd, GeneralConfig.Verbose); err != nil {
log.Entry().Error("failed running command codeql database create")
return err
}
return nil
}
func runDatabaseAnalyze(config *codeqlExecuteScanOptions, customFlags map[string]string, utils codeqlExecuteScanUtils) ([]piperutils.Path, error) {
sarifReport, err := executeAnalysis("sarif-latest", "codeqlReport.sarif", customFlags, config, utils)
if err != nil {
return nil, err
}
csvReport, err := executeAnalysis("csv", "codeqlReport.csv", customFlags, config, utils)
if err != nil {
return nil, err
}
return append(sarifReport, csvReport...), nil
}
func runGithubUploadResults(config *codeqlExecuteScanOptions, repoInfo *codeql.RepoInfo, token string, utils codeqlExecuteScanUtils) (string, error) {
cmd := prepareCmdForUploadResults(config, repoInfo, token)
var bufferOut, bufferErr bytes.Buffer
utils.Stdout(&bufferOut)
defer utils.Stdout(log.Writer())
utils.Stderr(&bufferErr)
defer utils.Stderr(log.Writer())
if err := execute(utils, cmd, GeneralConfig.Verbose); err != nil {
e := bufferErr.String()
log.Entry().Error(e)
if strings.Contains(e, "Unauthorized") {
log.Entry().Error("Either your Github Token is invalid or you use both Vault and Jenkins credentials where your Vault credentials are invalid, to use your Jenkins credentials try setting 'skipVault:true'")
}
return "", err
}
url := strings.TrimSpace(bufferOut.String())
return url, nil
}
func executeAnalysis(format, reportName string, customFlags map[string]string, config *codeqlExecuteScanOptions, utils codeqlExecuteScanUtils) ([]piperutils.Path, error) {
moduleTargetPath := filepath.Join(config.ModulePath, "target")
report := filepath.Join(moduleTargetPath, reportName)
cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, format, report)
if err != nil {
log.Entry().Errorf("failed to prepare command for codeql database analyze (format=%s)", format)
return nil, err
}
if err = execute(utils, cmd, GeneralConfig.Verbose); err != nil {
log.Entry().Errorf("failed running command codeql database analyze for %s generation", format)
return nil, err
}
return []piperutils.Path{
{Target: report},
}, nil
}
func prepareCmdForDatabaseCreate(customFlags map[string]string, config *codeqlExecuteScanOptions, utils codeqlExecuteScanUtils) ([]string, error) {
cmd := []string{"database", "create", config.Database}
cmd = codeql.AppendFlagIfNotSetByUser(cmd, []string{"--overwrite", "--no-overwrite"}, []string{"--overwrite"}, customFlags)
@ -430,7 +316,7 @@ func prepareCmdForDatabaseCreate(customFlags map[string]string, config *codeqlEx
}
func prepareCmdForDatabaseAnalyze(customFlags map[string]string, config *codeqlExecuteScanOptions, format, reportName string) ([]string, error) {
cmd := []string{"database", "analyze", "--format=" + format, fmt.Sprintf("--output=%v", filepath.Join(config.ModulePath, "target", reportName)), config.Database}
cmd := []string{"database", "analyze", "--format=" + format, "--output=" + reportName, config.Database}
cmd = codeql.AppendThreadsAndRam(cmd, config.Threads, config.Ram, customFlags)
cmd = codeql.AppendCustomFlags(cmd, customFlags)
cmd = appendCodeqlQuery(cmd, config.QuerySuite)
@ -459,16 +345,73 @@ func prepareCmdForUploadResults(config *codeqlExecuteScanOptions, repoInfo *code
cmd = append(cmd, "--repository="+(repoInfo.Owner+"/"+repoInfo.Repo))
}
if repoInfo.Ref != "" {
cmd = append(cmd, "--ref="+repoInfo.Ref)
if repoInfo.AnalyzedRef != "" {
cmd = append(cmd, "--ref="+repoInfo.AnalyzedRef)
}
return cmd
}
func addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite string, scanResults []codeql.CodeqlFindings, influx *codeqlExecuteScanInflux) {
influx.codeql_data.fields.repositoryURL = repoUrl
influx.codeql_data.fields.repositoryReferenceURL = repoRef
influx.codeql_data.fields.codeScanningLink = repoScanUrl
func uploadSarifResults(config *codeqlExecuteScanOptions, token string, repoInfo *codeql.RepoInfo, utils codeqlExecuteScanUtils) error {
sarifUrl, err := runGithubUploadResults(config, repoInfo, token, utils)
if err != nil {
return err
}
codeqlSarifUploader := codeql.NewCodeqlSarifUploaderInstance(sarifUrl, token)
err = codeql.WaitSarifUploaded(config.SarifCheckMaxRetries, config.SarifCheckRetryInterval, &codeqlSarifUploader)
if err != nil {
return errors.Wrap(err, "failed to upload sarif")
}
return nil
}
func uploadProjectToGitHub(config *codeqlExecuteScanOptions, repoInfo *codeql.RepoInfo) error {
log.Entry().Infof("DB sources for %s will be uploaded to target GitHub repo: %s", config.Repository, repoInfo.FullUrl)
hasToken, token := getToken(config)
if !hasToken {
return fmt.Errorf("failed running upload db sources to GitHub as githubToken was not specified")
}
repoUploader, err := codeql.NewGitUploaderInstance(
token,
repoInfo.AnalyzedRef,
config.Database,
repoInfo.CommitId,
config.Repository,
config.TargetGithubRepoURL,
)
if err != nil {
log.Entry().WithError(err).Error("failed to create github uploader")
return err
}
targetCommitId, err := repoUploader.UploadProjectToGithub()
if err != nil {
return errors.Wrap(err, "failed uploading db sources from non-GitHub SCM to GitHub")
}
repoInfo.CommitId = targetCommitId
log.Entry().Info("DB sources were successfully uploaded to target GitHub repo")
return nil
}
func checkForCompliance(scanResults []codeql.CodeqlFindings, config *codeqlExecuteScanOptions, repoInfo *codeql.RepoInfo) error {
for _, scanResult := range scanResults {
if scanResult.ClassificationName == codeql.AuditAll {
unaudited := scanResult.Total - scanResult.Audited
if unaudited > config.VulnerabilityThresholdTotal {
msg := fmt.Sprintf("Your repository %v with ref %v is not compliant. Total unaudited issues are %v which is greater than the VulnerabilityThresholdTotal count %v",
repoInfo.FullUrl, repoInfo.AnalyzedRef, unaudited, config.VulnerabilityThresholdTotal)
return errors.Errorf(msg)
}
}
}
return nil
}
func addDataToInfluxDB(repoInfo *codeql.RepoInfo, querySuite string, scanResults []codeql.CodeqlFindings, influx *codeqlExecuteScanInflux) {
influx.codeql_data.fields.repositoryURL = repoInfo.FullUrl
influx.codeql_data.fields.repositoryReferenceURL = repoInfo.FullRef
influx.codeql_data.fields.codeScanningLink = repoInfo.ScanUrl
influx.codeql_data.fields.querySuite = querySuite
for _, sr := range scanResults {

464
cmd/codeqlExecuteScan_test.go
View File

@ -4,14 +4,12 @@
package cmd
import (
"os"
"strings"
"testing"
"time"
"github.com/SAP/jenkins-library/pkg/codeql"
"github.com/SAP/jenkins-library/pkg/mock"
"github.com/SAP/jenkins-library/pkg/orchestrator"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
)
@ -30,281 +28,6 @@ func newCodeqlExecuteScanTestsUtils() codeqlExecuteScanMockUtils {
return utils
}
func TestRunCodeqlExecuteScan(t *testing.T) {
influx := &codeqlExecuteScanInflux{}
t.Run("Valid CodeqlExecuteScan", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "maven", ModulePath: "./"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.NoError(t, err)
})
t.Run("No auth token passed on upload results", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "maven", UploadResults: true, ModulePath: "./"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.Error(t, err)
})
t.Run("GitCommitID is NA on upload results", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "maven", UploadResults: true, ModulePath: "./", CommitID: "NA"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.Error(t, err)
})
t.Run("Custom buildtool", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "custom", Language: "javascript", ModulePath: "./"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.NoError(t, err)
})
t.Run("Custom buildtool but no language specified", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "custom", ModulePath: "./", GithubToken: "test"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.Error(t, err)
})
t.Run("Invalid buildtool and no language specified", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "test", ModulePath: "./", GithubToken: "test"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.Error(t, err)
})
t.Run("Invalid buildtool but language specified", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "test", Language: "javascript", ModulePath: "./", GithubToken: "test"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.NoError(t, err)
})
}
func TestGetGitRepoInfo(t *testing.T) {
t.Run("Valid https URL1", func(t *testing.T) {
var repoInfo codeql.RepoInfo
err := getGitRepoInfo("https://github.hello.test/Testing/fortify.git", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid https URL2", func(t *testing.T) {
var repoInfo codeql.RepoInfo
err := getGitRepoInfo("https://github.hello.test/Testing/fortify", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid https URL1 with dots", func(t *testing.T) {
var repoInfo codeql.RepoInfo
err := getGitRepoInfo("https://github.hello.test/Testing/com.sap.fortify.git", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "com.sap.fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid https URL2 with dots", func(t *testing.T) {
var repoInfo codeql.RepoInfo
err := getGitRepoInfo("https://github.hello.test/Testing/com.sap.fortify", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "com.sap.fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid https URL1 with username and token", func(t *testing.T) {
var repoInfo codeql.RepoInfo
err := getGitRepoInfo("https://username:token@github.hello.test/Testing/fortify.git", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid https URL2 with username and token", func(t *testing.T) {
var repoInfo codeql.RepoInfo
err := getGitRepoInfo("https://username:token@github.hello.test/Testing/fortify", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Invalid https URL as no org/Owner passed", func(t *testing.T) {
var repoInfo codeql.RepoInfo
assert.Error(t, getGitRepoInfo("https://github.com/fortify", &repoInfo))
})
t.Run("Invalid URL as no protocol passed", func(t *testing.T) {
var repoInfo codeql.RepoInfo
assert.Error(t, getGitRepoInfo("github.hello.test/Testing/fortify", &repoInfo))
})
t.Run("Valid ssh URL1", func(t *testing.T) {
var repoInfo codeql.RepoInfo
err := getGitRepoInfo("git@github.hello.test/Testing/fortify.git", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid ssh URL2", func(t *testing.T) {
var repoInfo codeql.RepoInfo
err := getGitRepoInfo("git@github.hello.test/Testing/fortify", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid ssh URL1 with dots", func(t *testing.T) {
var repoInfo codeql.RepoInfo
err := getGitRepoInfo("git@github.hello.test/Testing/com.sap.fortify.git", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "com.sap.fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid ssh URL2 with dots", func(t *testing.T) {
var repoInfo codeql.RepoInfo
err := getGitRepoInfo("git@github.hello.test/Testing/com.sap.fortify", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "com.sap.fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Invalid ssh URL as no org/Owner passed", func(t *testing.T) {
var repoInfo codeql.RepoInfo
assert.Error(t, getGitRepoInfo("git@github.com/fortify", &repoInfo))
})
}
func TestInitGitInfo(t *testing.T) {
t.Run("Valid URL1", func(t *testing.T) {
config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/codeql.git", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
repoInfo, err := initGitInfo(&config)
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "codeql", repoInfo.Repo)
assert.Equal(t, "refs/head/branch", repoInfo.Ref)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
})
t.Run("Valid URL2", func(t *testing.T) {
config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/codeql", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
repoInfo, err := initGitInfo(&config)
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "codeql", repoInfo.Repo)
assert.Equal(t, "refs/head/branch", repoInfo.Ref)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
})
t.Run("Valid url with dots URL1", func(t *testing.T) {
config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/com.sap.codeql.git", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
repoInfo, err := initGitInfo(&config)
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "com.sap.codeql", repoInfo.Repo)
assert.Equal(t, "refs/head/branch", repoInfo.Ref)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
})
t.Run("Valid url with dots URL2", func(t *testing.T) {
config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/com.sap.codeql", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
repoInfo, err := initGitInfo(&config)
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "com.sap.codeql", repoInfo.Repo)
assert.Equal(t, "refs/head/branch", repoInfo.Ref)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
})
t.Run("Valid url with username and token URL1", func(t *testing.T) {
config := codeqlExecuteScanOptions{Repository: "https://username:token@github.hello.test/Testing/codeql.git", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
repoInfo, err := initGitInfo(&config)
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "codeql", repoInfo.Repo)
assert.Equal(t, "refs/head/branch", repoInfo.Ref)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
})
t.Run("Valid url with username and token URL2", func(t *testing.T) {
config := codeqlExecuteScanOptions{Repository: "https://username:token@github.hello.test/Testing/codeql", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
repoInfo, err := initGitInfo(&config)
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "codeql", repoInfo.Repo)
assert.Equal(t, "refs/head/branch", repoInfo.Ref)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
})
t.Run("Invalid URL with no org/reponame", func(t *testing.T) {
config := codeqlExecuteScanOptions{Repository: "https://github.hello.test", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
repoInfo, err := initGitInfo(&config)
assert.NoError(t, err)
_, err = orchestrator.GetOrchestratorConfigProvider(nil)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "refs/head/branch", repoInfo.Ref)
if err != nil {
assert.Equal(t, "", repoInfo.Owner)
assert.Equal(t, "", repoInfo.Repo)
assert.Equal(t, "", repoInfo.ServerUrl)
}
})
}
func TestWaitSarifUploaded(t *testing.T) {
t.Parallel()
config := codeqlExecuteScanOptions{SarifCheckRetryInterval: 1, SarifCheckMaxRetries: 5}
t.Run("Fast complete upload", func(t *testing.T) {
codeqlScanAuditMock := CodeqlSarifUploaderMock{counter: 0}
timerStart := time.Now()
err := waitSarifUploaded(&config, &codeqlScanAuditMock)
assert.Less(t, time.Now().Sub(timerStart), time.Second)
assert.NoError(t, err)
})
t.Run("Long completed upload", func(t *testing.T) {
codeqlScanAuditMock := CodeqlSarifUploaderMock{counter: 2}
timerStart := time.Now()
err := waitSarifUploaded(&config, &codeqlScanAuditMock)
assert.GreaterOrEqual(t, time.Now().Sub(timerStart), time.Second*2)
assert.NoError(t, err)
})
t.Run("Failed upload", func(t *testing.T) {
codeqlScanAuditMock := CodeqlSarifUploaderMock{counter: -1}
err := waitSarifUploaded(&config, &codeqlScanAuditMock)
assert.Error(t, err)
assert.ErrorContains(t, err, "failed to upload sarif file")
})
t.Run("Error while checking sarif uploading", func(t *testing.T) {
codeqlScanAuditErrorMock := CodeqlSarifUploaderErrorMock{counter: -1}
err := waitSarifUploaded(&config, &codeqlScanAuditErrorMock)
assert.Error(t, err)
assert.ErrorContains(t, err, "test error")
})
t.Run("Completed upload after getting errors from server", func(t *testing.T) {
codeqlScanAuditErrorMock := CodeqlSarifUploaderErrorMock{counter: 3}
err := waitSarifUploaded(&config, &codeqlScanAuditErrorMock)
assert.NoError(t, err)
})
t.Run("Max retries reached", func(t *testing.T) {
codeqlScanAuditErrorMock := CodeqlSarifUploaderErrorMock{counter: 6}
err := waitSarifUploaded(&config, &codeqlScanAuditErrorMock)
assert.Error(t, err)
assert.ErrorContains(t, err, "max retries reached")
})
}
func TestGetMavenSettings(t *testing.T) {
t.Parallel()
t.Run("No maven", func(t *testing.T) {
@ -488,10 +211,16 @@ func TestAddDataToInfluxDB(t *testing.T) {
repoScanUrl := "https://github.htllo.test/Testing/codeql/security/code-scanning"
querySuite := "security.ql"
repoInfo := &codeql.RepoInfo{
FullUrl: repoUrl,
FullRef: repoRef,
ScanUrl: repoScanUrl,
}
t.Run("No findings", func(t *testing.T) {
scanResults := []codeql.CodeqlFindings{}
influx := &codeqlExecuteScanInflux{}
addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite, scanResults, influx)
addDataToInfluxDB(repoInfo, querySuite, scanResults, influx)
assert.Equal(t, repoUrl, influx.codeql_data.fields.repositoryURL)
assert.Equal(t, repoRef, influx.codeql_data.fields.repositoryReferenceURL)
assert.Equal(t, repoScanUrl, influx.codeql_data.fields.codeScanningLink)
@ -511,7 +240,7 @@ func TestAddDataToInfluxDB(t *testing.T) {
},
}
influx := &codeqlExecuteScanInflux{}
addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite, scanResults, influx)
addDataToInfluxDB(repoInfo, querySuite, scanResults, influx)
assert.Equal(t, repoUrl, influx.codeql_data.fields.repositoryURL)
assert.Equal(t, repoRef, influx.codeql_data.fields.repositoryReferenceURL)
assert.Equal(t, repoScanUrl, influx.codeql_data.fields.codeScanningLink)
@ -531,7 +260,7 @@ func TestAddDataToInfluxDB(t *testing.T) {
},
}
influx := &codeqlExecuteScanInflux{}
addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite, scanResults, influx)
addDataToInfluxDB(repoInfo, querySuite, scanResults, influx)
assert.Equal(t, repoUrl, influx.codeql_data.fields.repositoryURL)
assert.Equal(t, repoRef, influx.codeql_data.fields.repositoryReferenceURL)
assert.Equal(t, repoScanUrl, influx.codeql_data.fields.codeScanningLink)
@ -556,7 +285,7 @@ func TestAddDataToInfluxDB(t *testing.T) {
},
}
influx := &codeqlExecuteScanInflux{}
addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite, scanResults, influx)
addDataToInfluxDB(repoInfo, querySuite, scanResults, influx)
assert.Equal(t, repoUrl, influx.codeql_data.fields.repositoryURL)
assert.Equal(t, repoRef, influx.codeql_data.fields.repositoryReferenceURL)
assert.Equal(t, repoScanUrl, influx.codeql_data.fields.codeScanningLink)
@ -682,7 +411,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
config := &codeqlExecuteScanOptions{
Database: "codeqlDB",
}
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "codeqlReport.sarif")
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
assert.NoError(t, err)
assert.NotEmpty(t, cmd)
assert.Equal(t, 5, len(cmd))
@ -693,7 +422,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
config := &codeqlExecuteScanOptions{
Database: "codeqlDB",
}
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "csv", "codeqlReport.csv")
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "csv", "target/codeqlReport.csv")
assert.NoError(t, err)
assert.NotEmpty(t, cmd)
assert.Equal(t, 5, len(cmd))
@ -705,7 +434,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
Database: "codeqlDB",
QuerySuite: "security.ql",
}
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "codeqlReport.sarif")
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
assert.NoError(t, err)
assert.NotEmpty(t, cmd)
assert.Equal(t, 6, len(cmd))
@ -719,7 +448,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
Threads: "1",
Ram: "2000",
}
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "codeqlReport.sarif")
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
assert.NoError(t, err)
assert.NotEmpty(t, cmd)
assert.Equal(t, 8, len(cmd))
@ -736,7 +465,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
customFlags := map[string]string{
"--threads": "--threads=2",
}
cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "codeqlReport.sarif")
cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
assert.NoError(t, err)
assert.NotEmpty(t, cmd)
assert.Equal(t, 8, len(cmd))
@ -753,7 +482,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
customFlags := map[string]string{
"-j": "-j=2",
}
cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "codeqlReport.sarif")
cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
assert.NoError(t, err)
assert.NotEmpty(t, cmd)
assert.Equal(t, 8, len(cmd))
@ -770,7 +499,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
customFlags := map[string]string{
"--no-download": "--no-download",
}
cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "codeqlReport.sarif")
cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
assert.NoError(t, err)
assert.NotEmpty(t, cmd)
assert.Equal(t, 9, len(cmd))
@ -787,11 +516,11 @@ func TestPrepareCmdForUploadResults(t *testing.T) {
t.Run("All configs are set", func(t *testing.T) {
repoInfo := &codeql.RepoInfo{
CommitId: "commitId",
ServerUrl: "http://github.com",
Repo: "repo",
Owner: "owner",
Ref: "refs/heads/branch",
CommitId: "commitId",
ServerUrl: "http://github.com",
Repo: "repo",
Owner: "owner",
AnalyzedRef: "refs/heads/branch",
}
cmd := prepareCmdForUploadResults(config, repoInfo, "token")
assert.NotEmpty(t, cmd)
@ -811,11 +540,11 @@ func TestPrepareCmdForUploadResults(t *testing.T) {
t.Run("Empty token", func(t *testing.T) {
repoInfo := &codeql.RepoInfo{
CommitId: "commitId",
ServerUrl: "http://github.com",
Repo: "repo",
Owner: "owner",
Ref: "refs/heads/branch",
CommitId: "commitId",
ServerUrl: "http://github.com",
Repo: "repo",
Owner: "owner",
AnalyzedRef: "refs/heads/branch",
}
cmd := prepareCmdForUploadResults(config, repoInfo, "")
assert.NotEmpty(t, cmd)
@ -830,44 +559,115 @@ func TestPrepareCmdForUploadResults(t *testing.T) {
})
}
type CodeqlSarifUploaderMock struct {
counter int
func TestAppendCodeqlQuery(t *testing.T) {
t.Parallel()
t.Run("Empty query", func(t *testing.T) {
cmd := []string{"database", "analyze"}
query := ""
cmd = appendCodeqlQuery(cmd, query)
assert.Equal(t, 2, len(cmd))
})
t.Run("Not empty query", func(t *testing.T) {
cmd := []string{"database", "analyze"}
query := "java-extended.ql"
cmd = appendCodeqlQuery(cmd, query)
assert.Equal(t, 3, len(cmd))
})
}
func (c *CodeqlSarifUploaderMock) GetSarifStatus() (codeql.SarifFileInfo, error) {
if c.counter == 0 {
return codeql.SarifFileInfo{
ProcessingStatus: "complete",
Errors: nil,
}, nil
}
if c.counter == -1 {
return codeql.SarifFileInfo{
ProcessingStatus: "failed",
Errors: []string{"upload error"},
}, nil
}
c.counter--
return codeql.SarifFileInfo{
ProcessingStatus: "pending",
Errors: nil,
}, nil
func TestGetLangFromBuildTool(t *testing.T) {
t.Parallel()
t.Run("Build tool Maven", func(t *testing.T) {
assert.Equal(t, "java", getLangFromBuildTool("maven"))
})
t.Run("Build tool Pip", func(t *testing.T) {
assert.Equal(t, "python", getLangFromBuildTool("pip"))
})
t.Run("Build tool Npm", func(t *testing.T) {
assert.Equal(t, "javascript", getLangFromBuildTool("npm"))
})
t.Run("Build tool Yarn", func(t *testing.T) {
assert.Equal(t, "javascript", getLangFromBuildTool("yarn"))
})
t.Run("Build tool Golang", func(t *testing.T) {
assert.Equal(t, "go", getLangFromBuildTool("golang"))
})
t.Run("Build tool Unknown", func(t *testing.T) {
assert.Equal(t, "", getLangFromBuildTool("unknown"))
})
}
type CodeqlSarifUploaderErrorMock struct {
counter int
func TestGetToken(t *testing.T) {
t.Run("Token is set in config", func(t *testing.T) {
config := &codeqlExecuteScanOptions{GithubToken: "token"}
os.Setenv("GITHUB_TOKEN", "token_from_env")
hasToken, token := getToken(config)
os.Clearenv()
assert.True(t, hasToken)
assert.NotEmpty(t, token)
assert.Equal(t, "token", token)
})
t.Run("Token is set in env", func(t *testing.T) {
config := &codeqlExecuteScanOptions{}
os.Setenv("GITHUB_TOKEN", "token_from_env")
hasToken, token := getToken(config)
os.Clearenv()
assert.True(t, hasToken)
assert.NotEmpty(t, token)
assert.Equal(t, "token_from_env", token)
})
t.Run("Token is not set", func(t *testing.T) {
config := &codeqlExecuteScanOptions{}
hasToken, token := getToken(config)
assert.False(t, hasToken)
assert.Empty(t, token)
})
}
func (c *CodeqlSarifUploaderErrorMock) GetSarifStatus() (codeql.SarifFileInfo, error) {
if c.counter == -1 {
return codeql.SarifFileInfo{}, errors.New("test error")
func TestCheckForCompliance(t *testing.T) {
t.Parallel()
config := &codeqlExecuteScanOptions{VulnerabilityThresholdTotal: 0}
repoInfo := &codeql.RepoInfo{
FullUrl: "http://github.com/Test/repo",
AnalyzedRef: "refs/heads/branch",
}
if c.counter == 0 {
return codeql.SarifFileInfo{
ProcessingStatus: "complete",
Errors: nil,
}, nil
}
c.counter--
return codeql.SarifFileInfo{ProcessingStatus: "Service unavailable"}, nil
t.Run("Project is compliant", func(t *testing.T) {
scanResults := []codeql.CodeqlFindings{
{
ClassificationName: codeql.AuditAll,
Total: 10,
Audited: 10,
},
}
assert.NoError(t, checkForCompliance(scanResults, config, repoInfo))
})
t.Run("Project is not compliant", func(t *testing.T) {
scanResults := []codeql.CodeqlFindings{
{
ClassificationName: codeql.AuditAll,
Total: 20,
Audited: 10,
},
}
assert.Error(t, checkForCompliance(scanResults, config, repoInfo))
})
t.Run("Don't check Optional findings", func(t *testing.T) {
scanResults := []codeql.CodeqlFindings{
{
ClassificationName: codeql.Optional,
Total: 10,
Audited: 0,
},
}
assert.NoError(t, checkForCompliance(scanResults, config, repoInfo))
})
}

140
pkg/codeql/repo_info.go Normal file
View File

@ -0,0 +1,140 @@
package codeql
import (
"errors"
"fmt"
"regexp"
"strings"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/orchestrator"
)
type RepoInfo struct {
ServerUrl string
Owner string
Repo string
CommitId string
AnalyzedRef string
FullRef string
FullUrl string
ScanUrl string
}
func GetRepoInfo(repository, analyzedRef, commitID, targetGithubRepoURL, targetGithubBranchName string) (*RepoInfo, error) {
repoInfo := &RepoInfo{}
err := setRepoInfoFromRepoUri(repository, repoInfo)
if err != nil {
log.Entry().Error(err)
}
repoInfo.AnalyzedRef = analyzedRef
repoInfo.CommitId = commitID
getRepoInfoFromOrchestrator(repoInfo)
if len(targetGithubRepoURL) > 0 {
log.Entry().Infof("Checking target GitHub repo URL: %s", targetGithubRepoURL)
if err := setTargetGithubRepoInfo(targetGithubRepoURL, targetGithubBranchName, repoInfo); err != nil {
return repoInfo, err
}
}
repoUrl := fmt.Sprintf("%s/%s/%s", repoInfo.ServerUrl, repoInfo.Owner, repoInfo.Repo)
repoInfo.FullUrl = repoUrl
repoInfo.ScanUrl = fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.AnalyzedRef)
repoRef, err := buildRepoReference(repoUrl, repoInfo.AnalyzedRef)
if err != nil {
return nil, err
}
repoInfo.FullRef = repoRef
return repoInfo, nil
}
func buildRepoReference(repository, analyzedRef string) (string, error) {
ref := strings.Split(analyzedRef, "/")
if len(ref) < 3 {
return "", fmt.Errorf("wrong analyzedRef format: %s", analyzedRef)
}
if strings.Contains(analyzedRef, "pull") {
if len(ref) < 4 {
return "", fmt.Errorf("wrong analyzedRef format: %s", analyzedRef)
}
return fmt.Sprintf("%s/pull/%s", repository, ref[2]), nil
}
return fmt.Sprintf("%s/tree/%s", repository, ref[2]), nil
}
func setRepoInfoFromRepoUri(repoUri string, repoInfo *RepoInfo) error {
if repoUri == "" {
return errors.New("repository param is not set or it cannot be auto populated")
}
serverUrl, owner, repo, err := parseRepoUri(repoUri)
if err != nil {
return err
}
repoInfo.ServerUrl = serverUrl
repoInfo.Owner = owner
repoInfo.Repo = repo
return nil
}
func parseRepoUri(repoUri string) (string, string, string, error) {
pat := regexp.MustCompile(`^(https:\/\/|git@)([\S]+:[\S]+@)?([^\/:]+)[\/:]([^\/:]+\/[\S]+)$`)
matches := pat.FindAllStringSubmatch(repoUri, -1)
if len(matches) > 0 {
match := matches[0]
serverUrl := "https://" + match[3]
repoData := strings.Split(strings.TrimSuffix(match[4], ".git"), "/")
if len(repoData) != 2 {
return "", "", "", fmt.Errorf("invalid repository %s", repoUri)
}
owner, repo := repoData[0], repoData[1]
return serverUrl, owner, repo, nil
}
return "", "", "", fmt.Errorf("invalid repository %s", repoUri)
}
func getRepoInfoFromOrchestrator(repoInfo *RepoInfo) {
provider, err := orchestrator.GetOrchestratorConfigProvider(nil)
if err != nil {
log.Entry().Warn("No orchestrator found. We assume piper is running locally.")
} else {
if repoInfo.AnalyzedRef == "" {
repoInfo.AnalyzedRef = provider.GitReference()
}
if repoInfo.CommitId == "" || repoInfo.CommitId == "NA" {
repoInfo.CommitId = provider.CommitSHA()
}
if repoInfo.ServerUrl == "" {
err := setRepoInfoFromRepoUri(provider.RepoURL(), repoInfo)
if err != nil {
log.Entry().WithError(err).Error("failed to get repo info from orchestrator")
}
}
}
}
func setTargetGithubRepoInfo(targetGHRepoURL, targetGHBranchName string, repoInfo *RepoInfo) error {
if strings.Contains(repoInfo.ServerUrl, "github") {
return errors.New("TargetGithubRepoURL should not be set as the source repo is on github")
}
err := setRepoInfoFromRepoUri(targetGHRepoURL, repoInfo)
if err != nil {
log.Entry().WithError(err).Error("Failed to get target github repo info")
return err
}
if len(targetGHBranchName) > 0 {
log.Entry().Infof("Target GitHub branch name: %s", targetGHBranchName)
repoInfo.AnalyzedRef = getFullBranchName(targetGHBranchName)
}
return nil
}
func getFullBranchName(branchName string) string {
if len(strings.Split(branchName, "/")) < 3 {
return "refs/heads/" + branchName
}
return branchName
}

367
pkg/codeql/repo_info_test.go Normal file
View File

@ -0,0 +1,367 @@
package codeql
import (
"testing"
"github.com/SAP/jenkins-library/pkg/orchestrator"
"github.com/stretchr/testify/assert"
)
func TestGetRepoInfo(t *testing.T) {
t.Run("Valid URL1", func(t *testing.T) {
repo := "https://github.hello.test/Testing/codeql.git"
analyzedRef := "refs/heads/branch"
commitID := "abcd1234"
repoInfo, err := GetRepoInfo(repo, analyzedRef, commitID, "", "")
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "codeql", repoInfo.Repo)
assert.Equal(t, "refs/heads/branch", repoInfo.AnalyzedRef)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql", repoInfo.FullUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/security/code-scanning?query=is:open+ref:refs/heads/branch", repoInfo.ScanUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/tree/branch", repoInfo.FullRef)
})
t.Run("Valid URL2", func(t *testing.T) {
repo := "https://github.hello.test/Testing/codeql"
analyzedRef := "refs/heads/branch"
commitID := "abcd1234"
repoInfo, err := GetRepoInfo(repo, analyzedRef, commitID, "", "")
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "codeql", repoInfo.Repo)
assert.Equal(t, "refs/heads/branch", repoInfo.AnalyzedRef)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql", repoInfo.FullUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/security/code-scanning?query=is:open+ref:refs/heads/branch", repoInfo.ScanUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/tree/branch", repoInfo.FullRef)
})
t.Run("Valid url with dots URL1", func(t *testing.T) {
repo := "https://github.hello.test/Testing/com.sap.codeql.git"
analyzedRef := "refs/heads/branch"
commitID := "abcd1234"
repoInfo, err := GetRepoInfo(repo, analyzedRef, commitID, "", "")
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "com.sap.codeql", repoInfo.Repo)
assert.Equal(t, "refs/heads/branch", repoInfo.AnalyzedRef)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "https://github.hello.test/Testing/com.sap.codeql", repoInfo.FullUrl)
assert.Equal(t, "https://github.hello.test/Testing/com.sap.codeql/security/code-scanning?query=is:open+ref:refs/heads/branch", repoInfo.ScanUrl)
assert.Equal(t, "https://github.hello.test/Testing/com.sap.codeql/tree/branch", repoInfo.FullRef)
})
t.Run("Valid url with dots URL2", func(t *testing.T) {
repo := "https://github.hello.test/Testing/com.sap.codeql"
analyzedRef := "refs/heads/branch"
commitID := "abcd1234"
repoInfo, err := GetRepoInfo(repo, analyzedRef, commitID, "", "")
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "com.sap.codeql", repoInfo.Repo)
assert.Equal(t, "refs/heads/branch", repoInfo.AnalyzedRef)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "https://github.hello.test/Testing/com.sap.codeql", repoInfo.FullUrl)
assert.Equal(t, "https://github.hello.test/Testing/com.sap.codeql/security/code-scanning?query=is:open+ref:refs/heads/branch", repoInfo.ScanUrl)
assert.Equal(t, "https://github.hello.test/Testing/com.sap.codeql/tree/branch", repoInfo.FullRef)
})
t.Run("Valid url with username and token URL1", func(t *testing.T) {
repo := "https://username:token@github.hello.test/Testing/codeql.git"
analyzedRef := "refs/heads/branch"
commitID := "abcd1234"
repoInfo, err := GetRepoInfo(repo, analyzedRef, commitID, "", "")
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "codeql", repoInfo.Repo)
assert.Equal(t, "refs/heads/branch", repoInfo.AnalyzedRef)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql", repoInfo.FullUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/security/code-scanning?query=is:open+ref:refs/heads/branch", repoInfo.ScanUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/tree/branch", repoInfo.FullRef)
})
t.Run("Valid url with username and token URL2", func(t *testing.T) {
repo := "https://username:token@github.hello.test/Testing/codeql"
analyzedRef := "refs/heads/branch"
commitID := "abcd1234"
repoInfo, err := GetRepoInfo(repo, analyzedRef, commitID, "", "")
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "codeql", repoInfo.Repo)
assert.Equal(t, "refs/heads/branch", repoInfo.AnalyzedRef)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql", repoInfo.FullUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/security/code-scanning?query=is:open+ref:refs/heads/branch", repoInfo.ScanUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/tree/branch", repoInfo.FullRef)
})
t.Run("Invalid URL with no org/reponame", func(t *testing.T) {
repo := "https://github.hello.test"
analyzedRef := "refs/heads/branch"
commitID := "abcd1234"
repoInfo, err := GetRepoInfo(repo, analyzedRef, commitID, "", "")
assert.NoError(t, err)
_, err = orchestrator.GetOrchestratorConfigProvider(nil)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "refs/heads/branch", repoInfo.AnalyzedRef)
if err != nil {
assert.Equal(t, "", repoInfo.Owner)
assert.Equal(t, "", repoInfo.Repo)
assert.Equal(t, "", repoInfo.ServerUrl)
}
})
t.Run("Non-Github SCM, TargetGithubRepo is not empty", func(t *testing.T) {
repo := "https://gitlab.test/Testing/codeql.git"
analyzedRef := "refs/heads/branch"
commitID := "abcd1234"
targetGHRepoUrl := "https://github.hello.test/Testing/codeql"
repoInfo, err := GetRepoInfo(repo, analyzedRef, commitID, targetGHRepoUrl, "")
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "codeql", repoInfo.Repo)
assert.Equal(t, "refs/heads/branch", repoInfo.AnalyzedRef)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql", repoInfo.FullUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/security/code-scanning?query=is:open+ref:refs/heads/branch", repoInfo.ScanUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/tree/branch", repoInfo.FullRef)
})
t.Run("Non-Github SCM, TargetGithubRepo and TargetGithubBranch are not empty", func(t *testing.T) {
repo := "https://gitlab.test/Testing/codeql.git"
analyzedRef := "refs/heads/branch"
commitID := "abcd1234"
targetGHRepoUrl := "https://github.hello.test/Testing/codeql"
targetGHRepoBranch := "new-branch"
repoInfo, err := GetRepoInfo(repo, analyzedRef, commitID, targetGHRepoUrl, targetGHRepoBranch)
assert.NoError(t, err)
assert.Equal(t, "abcd1234", repoInfo.CommitId)
assert.Equal(t, "Testing", repoInfo.Owner)
assert.Equal(t, "codeql", repoInfo.Repo)
assert.Equal(t, "refs/heads/new-branch", repoInfo.AnalyzedRef)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql", repoInfo.FullUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/security/code-scanning?query=is:open+ref:refs/heads/new-branch", repoInfo.ScanUrl)
assert.Equal(t, "https://github.hello.test/Testing/codeql/tree/new-branch", repoInfo.FullRef)
})
}
func TestBuildRepoReference(t *testing.T) {
t.Run("Valid AnalyzedRef with branch", func(t *testing.T) {
repo := "https://github.hello.test/Testing/fortify"
analyzedRef := "refs/heads/branch"
ref, err := buildRepoReference(repo, analyzedRef)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test/Testing/fortify/tree/branch", ref)
})
t.Run("Valid AnalyzedRef with PR", func(t *testing.T) {
repo := "https://github.hello.test/Testing/fortify"
analyzedRef := "refs/pull/1/merge"
ref, err := buildRepoReference(repo, analyzedRef)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test/Testing/fortify/pull/1", ref)
})
t.Run("Invalid AnalyzedRef without branch name", func(t *testing.T) {
repo := "https://github.hello.test/Testing/fortify"
analyzedRef := "refs/heads"
ref, err := buildRepoReference(repo, analyzedRef)
assert.Error(t, err)
assert.ErrorContains(t, err, "wrong analyzedRef format")
assert.Equal(t, "", ref)
})
t.Run("Invalid AnalyzedRef without PR id", func(t *testing.T) {
repo := "https://github.hello.test/Testing/fortify"
analyzedRef := "refs/pull/merge"
ref, err := buildRepoReference(repo, analyzedRef)
assert.Error(t, err)
assert.ErrorContains(t, err, "wrong analyzedRef format")
assert.Equal(t, "", ref)
})
}
func TestSetRepoInfoFromRepoUri(t *testing.T) {
t.Run("Valid https URL1", func(t *testing.T) {
var repoInfo RepoInfo
err := setRepoInfoFromRepoUri("https://github.hello.test/Testing/fortify.git", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid https URL2", func(t *testing.T) {
var repoInfo RepoInfo
err := setRepoInfoFromRepoUri("https://github.hello.test/Testing/fortify", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid https URL1 with dots", func(t *testing.T) {
var repoInfo RepoInfo
err := setRepoInfoFromRepoUri("https://github.hello.test/Testing/com.sap.fortify.git", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "com.sap.fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid https URL2 with dots", func(t *testing.T) {
var repoInfo RepoInfo
err := setRepoInfoFromRepoUri("https://github.hello.test/Testing/com.sap.fortify", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "com.sap.fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid https URL1 with username and token", func(t *testing.T) {
var repoInfo RepoInfo
err := setRepoInfoFromRepoUri("https://username:token@github.hello.test/Testing/fortify.git", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid https URL2 with username and token", func(t *testing.T) {
var repoInfo RepoInfo
err := setRepoInfoFromRepoUri("https://username:token@github.hello.test/Testing/fortify", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Invalid https URL as no org/Owner passed", func(t *testing.T) {
var repoInfo RepoInfo
assert.Error(t, setRepoInfoFromRepoUri("https://github.com/fortify", &repoInfo))
})
t.Run("Invalid URL as no protocol passed", func(t *testing.T) {
var repoInfo RepoInfo
assert.Error(t, setRepoInfoFromRepoUri("github.hello.test/Testing/fortify", &repoInfo))
})
t.Run("Valid ssh URL1", func(t *testing.T) {
var repoInfo RepoInfo
err := setRepoInfoFromRepoUri("git@github.hello.test/Testing/fortify.git", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid ssh URL2", func(t *testing.T) {
var repoInfo RepoInfo
err := setRepoInfoFromRepoUri("git@github.hello.test/Testing/fortify", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid ssh URL1 with dots", func(t *testing.T) {
var repoInfo RepoInfo
err := setRepoInfoFromRepoUri("git@github.hello.test/Testing/com.sap.fortify.git", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "com.sap.fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Valid ssh URL2 with dots", func(t *testing.T) {
var repoInfo RepoInfo
err := setRepoInfoFromRepoUri("git@github.hello.test/Testing/com.sap.fortify", &repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test", repoInfo.ServerUrl)
assert.Equal(t, "com.sap.fortify", repoInfo.Repo)
assert.Equal(t, "Testing", repoInfo.Owner)
})
t.Run("Invalid ssh URL as no org/Owner passed", func(t *testing.T) {
var repoInfo RepoInfo
assert.Error(t, setRepoInfoFromRepoUri("git@github.com/fortify", &repoInfo))
})
}
func TestSetTargetGithubRepoInfo(t *testing.T) {
t.Parallel()
t.Run("Source repo server is github", func(t *testing.T) {
repoInfo := &RepoInfo{
ServerUrl: "https://github.com",
Owner: "owner",
Repo: "repo",
}
targetRepo := "https://github.com/target/repo"
targetBranch := "target-branch"
err := setTargetGithubRepoInfo(targetRepo, targetBranch, repoInfo)
assert.Error(t, err)
})
t.Run("Success", func(t *testing.T) {
repoInfo := &RepoInfo{
ServerUrl: "https://gitlab.com",
Owner: "owner",
Repo: "repo",
AnalyzedRef: "refs/heads/source-branch",
}
targetRepo := "https://github.com/target/repo"
targetBranch := "target-branch"
err := setTargetGithubRepoInfo(targetRepo, targetBranch, repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.com", repoInfo.ServerUrl)
assert.Equal(t, "target", repoInfo.Owner)
assert.Equal(t, "repo", repoInfo.Repo)
assert.Equal(t, "refs/heads/target-branch", repoInfo.AnalyzedRef)
})
t.Run("Empty target branch", func(t *testing.T) {
repoInfo := &RepoInfo{
ServerUrl: "https://gitlab.com",
Owner: "owner",
Repo: "repo",
AnalyzedRef: "refs/heads/source-branch",
}
targetRepo := "https://github.com/target/repo"
err := setTargetGithubRepoInfo(targetRepo, "", repoInfo)
assert.NoError(t, err)
assert.Equal(t, "https://github.com", repoInfo.ServerUrl)
assert.Equal(t, "target", repoInfo.Owner)
assert.Equal(t, "repo", repoInfo.Repo)
assert.Equal(t, "refs/heads/source-branch", repoInfo.AnalyzedRef)
})
}
func TestGetFullBranchName(t *testing.T) {
t.Parallel()
t.Run("Given short branch name", func(t *testing.T) {
input := "branch-name"
assert.Equal(t, "refs/heads/branch-name", getFullBranchName(input))
})
t.Run("Given full branch name", func(t *testing.T) {
input := "refs/heads/branch-name"
assert.Equal(t, "refs/heads/branch-name", getFullBranchName(input))
})
}

49
pkg/codeql/reporting.go
View File

@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"path/filepath"
"strings"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
@ -27,14 +26,6 @@ type CodeqlFindings struct {
Audited int `json:"audited"`
}
type RepoInfo struct {
ServerUrl string
Repo string
CommitId string
Ref string
Owner string
}
func WriteJSONReport(jsonReport CodeqlAudit, modulePath string) ([]piperutils.Path, error) {
utils := piperutils.Files{}
reportPaths := []piperutils.Path{}
@ -56,22 +47,8 @@ func WriteJSONReport(jsonReport CodeqlAudit, modulePath string) ([]piperutils.Pa
return reportPaths, nil
}
func BuildRepoReference(repository, analyzedRef string) (string, error) {
ref := strings.Split(analyzedRef, "/")
if len(ref) < 3 {
return "", errors.New(fmt.Sprintf("Wrong analyzedRef format: %s", analyzedRef))
}
if strings.Contains(analyzedRef, "pull") {
if len(ref) < 4 {
return "", errors.New(fmt.Sprintf("Wrong analyzedRef format: %s", analyzedRef))
}
return fmt.Sprintf("%s/pull/%s", repository, ref[2]), nil
}
return fmt.Sprintf("%s/tree/%s", repository, ref[2]), nil
}
func CreateAndPersistToolRecord(utils piperutils.FileUtils, repoInfo RepoInfo, repoReference, repoUrl, modulePath string) (string, error) {
toolRecord, err := createToolRecordCodeql(utils, repoInfo, repoReference, repoUrl, modulePath)
func CreateAndPersistToolRecord(utils piperutils.FileUtils, repoInfo *RepoInfo, modulePath string) (string, error) {
toolRecord, err := createToolRecordCodeql(utils, repoInfo, modulePath)
if err != nil {
return "", err
}
@ -84,7 +61,7 @@ func CreateAndPersistToolRecord(utils piperutils.FileUtils, repoInfo RepoInfo, r
return toolRecordFileName, nil
}
func createToolRecordCodeql(utils piperutils.FileUtils, repoInfo RepoInfo, repoUrl, repoReference, modulePath string) (*toolrecord.Toolrecord, error) {
func createToolRecordCodeql(utils piperutils.FileUtils, repoInfo *RepoInfo, modulePath string) (*toolrecord.Toolrecord, error) {
record := toolrecord.New(utils, modulePath, "codeql", repoInfo.ServerUrl)
if repoInfo.ServerUrl == "" {
@ -95,33 +72,33 @@ func createToolRecordCodeql(utils piperutils.FileUtils, repoInfo RepoInfo, repoU
return record, errors.New("CommitId not set")
}
if repoInfo.Ref == "" {
if repoInfo.AnalyzedRef == "" {
return record, errors.New("Analyzed Reference not set")
}
record.DisplayName = fmt.Sprintf("%s %s - %s %s", repoInfo.Owner, repoInfo.Repo, repoInfo.Ref, repoInfo.CommitId)
record.DisplayURL = fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.Ref)
record.DisplayName = fmt.Sprintf("%s %s - %s %s", repoInfo.Owner, repoInfo.Repo, repoInfo.AnalyzedRef, repoInfo.CommitId)
record.DisplayURL = repoInfo.ScanUrl
err := record.AddKeyData("repository",
fmt.Sprintf("%s/%s", repoInfo.Owner, repoInfo.Repo),
fmt.Sprintf("%s %s", repoInfo.Owner, repoInfo.Repo),
repoUrl)
repoInfo.FullUrl)
if err != nil {
return record, err
}
err = record.AddKeyData("repositoryReference",
repoInfo.Ref,
fmt.Sprintf("%s - %s", repoInfo.Repo, repoInfo.Ref),
repoReference)
repoInfo.AnalyzedRef,
fmt.Sprintf("%s - %s", repoInfo.Repo, repoInfo.AnalyzedRef),
repoInfo.FullRef)
if err != nil {
return record, err
}
err = record.AddKeyData("scanResult",
fmt.Sprintf("%s/%s", repoInfo.Ref, repoInfo.CommitId),
fmt.Sprintf("%s %s - %s %s", repoInfo.Owner, repoInfo.Repo, repoInfo.Ref, repoInfo.CommitId),
fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.Ref))
fmt.Sprintf("%s/%s", repoInfo.AnalyzedRef, repoInfo.CommitId),
fmt.Sprintf("%s %s - %s %s", repoInfo.Owner, repoInfo.Repo, repoInfo.AnalyzedRef, repoInfo.CommitId),
repoInfo.ScanUrl)
if err != nil {
return record, err
}

80
pkg/codeql/reporting_test.go
View File

@ -1,7 +1,6 @@
package codeql
import (
"fmt"
"testing"
"github.com/SAP/jenkins-library/pkg/mock"
@ -21,87 +20,54 @@ func newCodeqlExecuteScanTestsUtils() codeqlExecuteScanMockUtils {
return utils
}
func TestBuildRepoReference(t *testing.T) {
t.Run("Valid Ref with branch", func(t *testing.T) {
repository := "https://github.hello.test/Testing/fortify"
analyzedRef := "refs/head/branch"
ref, err := BuildRepoReference(repository, analyzedRef)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test/Testing/fortify/tree/branch", ref)
})
t.Run("Valid Ref with PR", func(t *testing.T) {
repository := "https://github.hello.test/Testing/fortify"
analyzedRef := "refs/pull/1/merge"
ref, err := BuildRepoReference(repository, analyzedRef)
assert.NoError(t, err)
assert.Equal(t, "https://github.hello.test/Testing/fortify/pull/1", ref)
})
t.Run("Invalid Ref without branch name", func(t *testing.T) {
repository := "https://github.hello.test/Testing/fortify"
analyzedRef := "refs/head"
ref, err := BuildRepoReference(repository, analyzedRef)
assert.Error(t, err)
assert.ErrorContains(t, err, "Wrong analyzedRef format")
assert.Equal(t, "", ref)
})
t.Run("Invalid Ref without PR id", func(t *testing.T) {
repository := "https://github.hello.test/Testing/fortify"
analyzedRef := "refs/pull/merge"
ref, err := BuildRepoReference(repository, analyzedRef)
assert.Error(t, err)
assert.ErrorContains(t, err, "Wrong analyzedRef format")
assert.Equal(t, "", ref)
})
}
func getRepoReferences(repoInfo RepoInfo) (string, string) {
repoUrl := fmt.Sprintf("%s/%s/%s", repoInfo.ServerUrl, repoInfo.Owner, repoInfo.Repo)
repoReference, _ := BuildRepoReference(repoUrl, repoInfo.Ref)
return repoUrl, repoReference
}
func TestCreateToolRecordCodeql(t *testing.T) {
modulePath := "./"
t.Run("Valid toolrun file", func(t *testing.T) {
repoInfo := RepoInfo{ServerUrl: "https://github.hello.test", CommitId: "test", Ref: "refs/head/branch", Owner: "Testing", Repo: "fortify"}
repoUrl, repoReference := getRepoReferences(repoInfo)
toolRecord, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, modulePath)
repoInfo := &RepoInfo{
ServerUrl: "https://github.hello.test",
CommitId: "test",
AnalyzedRef: "refs/heads/branch",
Owner: "Testing",
Repo: "codeql",
FullUrl: "https://github.hello.test/Testing/codeql",
FullRef: "https://github.hello.test/Testing/codeql/tree/branch",
ScanUrl: "https://github.hello.test/Testing/codeql/security/code-scanning?query=is:open+ref:refs/heads/branch",
}
toolRecord, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, modulePath)
assert.NoError(t, err)
assert.Equal(t, toolRecord.ToolName, "codeql")
assert.Equal(t, toolRecord.ToolInstance, "https://github.hello.test")
assert.Equal(t, toolRecord.DisplayName, "Testing fortify - refs/head/branch test")
assert.Equal(t, toolRecord.DisplayURL, "https://github.hello.test/Testing/fortify/security/code-scanning?query=is:open+ref:refs/head/branch")
assert.Equal(t, toolRecord.DisplayName, "Testing codeql - refs/heads/branch test")
assert.Equal(t, toolRecord.DisplayURL, "https://github.hello.test/Testing/codeql/security/code-scanning?query=is:open+ref:refs/heads/branch")
})
t.Run("Empty repository URL", func(t *testing.T) {
repoInfo := RepoInfo{ServerUrl: "", CommitId: "test", Ref: "refs/head/branch", Owner: "Testing", Repo: "fortify"}
repoUrl, repoReference := getRepoReferences(repoInfo)
_, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, modulePath)
repoInfo := &RepoInfo{ServerUrl: "", CommitId: "test", AnalyzedRef: "refs/heads/branch", Owner: "Testing", Repo: "codeql"}
_, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, modulePath)
assert.Error(t, err)
assert.ErrorContains(t, err, "Repository not set")
})
t.Run("Empty analyzedRef", func(t *testing.T) {
repoInfo := RepoInfo{ServerUrl: "https://github.hello.test", CommitId: "test", Ref: "", Owner: "Testing", Repo: "fortify"}
repoUrl, repoReference := getRepoReferences(repoInfo)
_, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, modulePath)
repoInfo := &RepoInfo{ServerUrl: "https://github.hello.test", CommitId: "test", AnalyzedRef: "", Owner: "Testing", Repo: "codeql"}
_, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, modulePath)
assert.Error(t, err)
assert.ErrorContains(t, err, "Analyzed Reference not set")
})
t.Run("Empty CommitId", func(t *testing.T) {
repoInfo := RepoInfo{ServerUrl: "https://github.hello.test", CommitId: "", Ref: "refs/head/branch", Owner: "Testing", Repo: "fortify"}
repoUrl, repoReference := getRepoReferences(repoInfo)
_, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, modulePath)
repoInfo := &RepoInfo{ServerUrl: "https://github.hello.test", CommitId: "", AnalyzedRef: "refs/heads/branch", Owner: "Testing", Repo: "codeql"}
_, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, modulePath)
assert.Error(t, err)
assert.ErrorContains(t, err, "CommitId not set")
})
t.Run("Invalid analyzedRef", func(t *testing.T) {
repoInfo := RepoInfo{ServerUrl: "https://github.hello.test", CommitId: "", Ref: "refs/branch", Owner: "Testing", Repo: "fortify"}
repoUrl, repoReference := getRepoReferences(repoInfo)
_, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, repoUrl, repoReference, modulePath)
repoInfo := &RepoInfo{ServerUrl: "https://github.hello.test", CommitId: "", AnalyzedRef: "refs/branch", Owner: "Testing", Repo: "codeql"}
_, err := createToolRecordCodeql(newCodeqlExecuteScanTestsUtils(), repoInfo, modulePath)
assert.Error(t, err)
})

39
pkg/codeql/sarif_upload.go
View File

@ -2,8 +2,17 @@ package codeql
import (
"encoding/json"
"errors"
"io"
"net/http"
"time"
"github.com/SAP/jenkins-library/pkg/log"
)
const (
sarifUploadComplete = "complete"
sarifUploadFailed = "failed"
)
type CodeqlSarifUploader interface {
@ -66,3 +75,33 @@ func getSarifUploadingStatus(sarifURL, token string) (SarifFileInfo, error) {
}
return sarifInfo, nil
}
func WaitSarifUploaded(maxRetries, checkRetryInterval int, codeqlSarifUploader CodeqlSarifUploader) error {
retryInterval := time.Duration(checkRetryInterval) * time.Second
log.Entry().Info("waiting for the SARIF to upload")
i := 1
for {
sarifStatus, err := codeqlSarifUploader.GetSarifStatus()
if err != nil {
return err
}
log.Entry().Infof("the SARIF processing status: %s", sarifStatus.ProcessingStatus)
if sarifStatus.ProcessingStatus == sarifUploadComplete {
return nil
}
if sarifStatus.ProcessingStatus == sarifUploadFailed {
for e := range sarifStatus.Errors {
log.Entry().Error(e)
}
return errors.New("failed to upload sarif file")
}
if i <= maxRetries {
log.Entry().Infof("still waiting for the SARIF to upload: retrying in %d seconds... (retry %d/%d)", checkRetryInterval, i, maxRetries)
time.Sleep(retryInterval)
i++
continue
}
return errors.New("failed to check sarif uploading status: max retries reached")
}
}

94
pkg/codeql/sarif_upload_test.go Normal file
View File

@ -0,0 +1,94 @@
package codeql
import (
"testing"
"time"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
)
type CodeqlSarifUploaderMock struct {
counter int
}
func (c *CodeqlSarifUploaderMock) GetSarifStatus() (SarifFileInfo, error) {
if c.counter == 0 {
return SarifFileInfo{
ProcessingStatus: "complete",
Errors: nil,
}, nil
}
if c.counter == -1 {
return SarifFileInfo{
ProcessingStatus: "failed",
Errors: []string{"upload error"},
}, nil
}
c.counter--
return SarifFileInfo{
ProcessingStatus: "pending",
Errors: nil,
}, nil
}
type CodeqlSarifUploaderErrorMock struct {
counter int
}
func (c *CodeqlSarifUploaderErrorMock) GetSarifStatus() (SarifFileInfo, error) {
if c.counter == -1 {
return SarifFileInfo{}, errors.New("test error")
}
if c.counter == 0 {
return SarifFileInfo{
ProcessingStatus: "complete",
Errors: nil,
}, nil
}
c.counter--
return SarifFileInfo{ProcessingStatus: "Service unavailable"}, nil
}
func TestWaitSarifUploaded(t *testing.T) {
t.Parallel()
sarifCheckRetryInterval := 1
sarifCheckMaxRetries := 5
t.Run("Fast complete upload", func(t *testing.T) {
codeqlScanAuditMock := CodeqlSarifUploaderMock{counter: 0}
timerStart := time.Now()
err := WaitSarifUploaded(sarifCheckMaxRetries, sarifCheckRetryInterval, &codeqlScanAuditMock)
assert.Less(t, time.Now().Sub(timerStart), time.Second)
assert.NoError(t, err)
})
t.Run("Long completed upload", func(t *testing.T) {
codeqlScanAuditMock := CodeqlSarifUploaderMock{counter: 2}
timerStart := time.Now()
err := WaitSarifUploaded(sarifCheckMaxRetries, sarifCheckRetryInterval, &codeqlScanAuditMock)
assert.GreaterOrEqual(t, time.Now().Sub(timerStart), time.Second*2)
assert.NoError(t, err)
})
t.Run("Failed upload", func(t *testing.T) {
codeqlScanAuditMock := CodeqlSarifUploaderMock{counter: -1}
err := WaitSarifUploaded(sarifCheckMaxRetries, sarifCheckRetryInterval, &codeqlScanAuditMock)
assert.Error(t, err)
assert.ErrorContains(t, err, "failed to upload sarif file")
})
t.Run("Error while checking sarif uploading", func(t *testing.T) {
codeqlScanAuditErrorMock := CodeqlSarifUploaderErrorMock{counter: -1}
err := WaitSarifUploaded(sarifCheckMaxRetries, sarifCheckRetryInterval, &codeqlScanAuditErrorMock)
assert.Error(t, err)
assert.ErrorContains(t, err, "test error")
})
t.Run("Completed upload after getting errors from server", func(t *testing.T) {
codeqlScanAuditErrorMock := CodeqlSarifUploaderErrorMock{counter: 3}
err := WaitSarifUploaded(sarifCheckMaxRetries, sarifCheckRetryInterval, &codeqlScanAuditErrorMock)
assert.NoError(t, err)
})
t.Run("Max retries reached", func(t *testing.T) {
codeqlScanAuditErrorMock := CodeqlSarifUploaderErrorMock{counter: 6}
err := WaitSarifUploaded(sarifCheckMaxRetries, sarifCheckRetryInterval, &codeqlScanAuditErrorMock)
assert.Error(t, err)
assert.ErrorContains(t, err, "max retries reached")
})
}