You've already forked sap-jenkins-library
mirror of
https://github.com/SAP/jenkins-library.git
synced 2025-09-16 09:26:22 +02:00
Merge branch 'master' into phgermanov/error-patterns-top-10
This commit is contained in:
28
.github/renovate.json
vendored
28
.github/renovate.json
vendored
@@ -104,6 +104,34 @@
|
||||
"depNameTemplate": "CycloneDX/{{{depName}}}",
|
||||
"extractVersionTemplate": "v(?<version>.*)",
|
||||
"autoReplaceStringTemplate": "\"github.com/{{{depName}}}/cmd/cyclonedx-gomod@v{{{newValue}}}\""
|
||||
},
|
||||
{
|
||||
"customType": "regex",
|
||||
"matchStringsStrategy": "any",
|
||||
"managerFilePatterns": [
|
||||
"resources/metadata/pythonBuild.yaml"
|
||||
],
|
||||
"matchStrings": [
|
||||
"image: (?<depName>python):(?<currentValue>.*)"
|
||||
],
|
||||
"depTypeTemplate": "dependencies",
|
||||
"datasourceTemplate": "docker",
|
||||
"extractVersionTemplate": "(?<version>.*)",
|
||||
"autoReplaceStringTemplate": "image: {{depName}}:{{newValue}}"
|
||||
},
|
||||
{
|
||||
"customType": "regex",
|
||||
"matchStringsStrategy": "any",
|
||||
"managerFilePatterns": [
|
||||
"cmd/pythonBuild_generated.go"
|
||||
],
|
||||
"matchStrings": [
|
||||
"Image: \"(?<depName>python):(?<currentValue>.*)\""
|
||||
],
|
||||
"depTypeTemplate": "dependencies",
|
||||
"datasourceTemplate": "docker",
|
||||
"extractVersionTemplate": "(?<version>.*)",
|
||||
"autoReplaceStringTemplate": "Image: \"{{depName}}:{{newValue}}\""
|
||||
}
|
||||
],
|
||||
"postUpdateOptions": [
|
||||
|
@@ -128,7 +128,8 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS
|
||||
}
|
||||
}
|
||||
|
||||
scans, err := cx1sh.GetLastScans(10)
|
||||
branch, isPR, baseBranch := cx1sh.GetScanBranch()
|
||||
scans, err := cx1sh.GetLastScans(10, branch)
|
||||
if err != nil {
|
||||
log.Entry().WithError(err).Warnf("failed to get last 10 scans")
|
||||
}
|
||||
@@ -158,7 +159,7 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS
|
||||
}
|
||||
|
||||
if config.Incremental {
|
||||
log.Entry().Warnf("If you change your file filter pattern it is recommended to run a Full scan instead of an incremental, to ensure full code coverage.")
|
||||
log.Entry().Info("If you change your file filter pattern it is recommended to run a Full scan instead of an incremental, to ensure full code coverage.")
|
||||
}
|
||||
|
||||
zipFile, err := cx1sh.ZipFiles()
|
||||
@@ -172,7 +173,40 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS
|
||||
}
|
||||
|
||||
// TODO : The step structure should allow to enable different scanners: SAST, KICKS, SCA
|
||||
scan, err := cx1sh.CreateScanRequest(incremental, uploadLink)
|
||||
var scan *checkmarxOne.Scan
|
||||
// We search for a full scan in history:
|
||||
fullScanExists := false
|
||||
for _, histScan := range scans {
|
||||
isIncr, _ := histScan.IsIncremental()
|
||||
if isIncr == false {
|
||||
fullScanExists = true
|
||||
break
|
||||
}
|
||||
}
|
||||
// user requested an incremental scan on a branch, and the project has a Primary Branch set, not in PR context and no full scan on the branch
|
||||
if config.Incremental && !isPR && !fullScanExists && cx1sh.Project.MainBranch != "" && cx1sh.Project.MainBranch != branch {
|
||||
scansMainBranch, err := cx1sh.GetLastScans(10, cx1sh.Project.MainBranch)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to determine incremental or full scan configuration: %s", err)
|
||||
}
|
||||
// We check if the main branch is eligible for an incremental scan
|
||||
incrementalMainBranch, err := cx1sh.IncrementalOrFull(scansMainBranch)
|
||||
log.Entry().Debugf("Main branch %v incremental scan eligibility: %t", cx1sh.Project.MainBranch, incrementalMainBranch)
|
||||
scan, err = cx1sh.CreateScanRequest(incrementalMainBranch, uploadLink, cx1sh.Project.MainBranch) // this will create a full scan on the current branch if the main branch is not eligible for an incremental scan
|
||||
} else if config.Incremental && isPR && len(baseBranch) > 0 && baseBranch != "n/a" { // running in a PR context, and we have a base branch for the incremental scan
|
||||
// in a PR context we always want to do an incremental scan
|
||||
// The scan will be based on the PR's target branch (baseBranch) if there is no full scan on the PR branch
|
||||
if fullScanExists {
|
||||
log.Entry().Debugf("A full scan exists on the PR branch %v, so the incremental scan will be based on it", branch)
|
||||
scan, err = cx1sh.CreateScanRequest(true, uploadLink, "")
|
||||
} else {
|
||||
log.Entry().Debugf("There is no full scan on the PR branch %v, so the incremental scan will be based on branch %v", branch, baseBranch)
|
||||
scan, err = cx1sh.CreateScanRequest(true, uploadLink, baseBranch)
|
||||
}
|
||||
} else {
|
||||
scan, err = cx1sh.CreateScanRequest(incremental, uploadLink, "")
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create scan: %s", err)
|
||||
}
|
||||
@@ -388,8 +422,8 @@ func (c *checkmarxOneExecuteScanHelper) SetProjectPreset() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *checkmarxOneExecuteScanHelper) GetLastScans(count int) ([]checkmarxOne.Scan, error) {
|
||||
scans, err := c.sys.GetLastScansByStatus(c.Project.ProjectID, count, []string{"Completed"})
|
||||
func (c *checkmarxOneExecuteScanHelper) GetLastScans(count int, branch string) ([]checkmarxOne.Scan, error) {
|
||||
scans, err := c.sys.GetLastScansByStatus(c.Project.ProjectID, branch, count, []string{"Completed"})
|
||||
if err != nil {
|
||||
return []checkmarxOne.Scan{}, fmt.Errorf("Failed to get last %d Completed scans for project %v: %s", count, c.Project.ProjectID, err)
|
||||
}
|
||||
@@ -437,23 +471,10 @@ func (c *checkmarxOneExecuteScanHelper) UploadScanContent(zipFile *os.File) (str
|
||||
return uploadUri, nil
|
||||
}
|
||||
|
||||
func (c *checkmarxOneExecuteScanHelper) CreateScanRequest(incremental bool, uploadLink string) (*checkmarxOne.Scan, error) {
|
||||
sastConfig := checkmarxOne.ScanConfiguration{}
|
||||
sastConfig.ScanType = "sast"
|
||||
|
||||
sastConfig.Values = make(map[string]string, 0)
|
||||
sastConfig.Values["incremental"] = strconv.FormatBool(incremental)
|
||||
sastConfig.Values["presetName"] = c.config.Preset // always set, either coming from config or coming from Cx1 configuration
|
||||
sastConfigString := fmt.Sprintf("incremental %v, preset %v", strconv.FormatBool(incremental), c.config.Preset)
|
||||
|
||||
if len(c.config.LanguageMode) > 0 {
|
||||
sastConfig.Values["languageMode"] = c.config.LanguageMode
|
||||
sastConfigString = sastConfigString + fmt.Sprintf(", languageMode %v", c.config.LanguageMode)
|
||||
}
|
||||
|
||||
func (c *checkmarxOneExecuteScanHelper) GetScanBranch() (string, bool, string) {
|
||||
branch := c.config.Branch
|
||||
cicdOrch, err := orchestrator.GetOrchestratorConfigProvider(nil)
|
||||
if err == nil {
|
||||
if err != nil {
|
||||
log.Entry().Warn("Could not identify orchestrator")
|
||||
}
|
||||
if len(branch) == 0 && len(c.config.GitBranch) > 0 && c.config.GitBranch != "n/a" {
|
||||
@@ -462,17 +483,43 @@ func (c *checkmarxOneExecuteScanHelper) CreateScanRequest(incremental bool, uplo
|
||||
cicdBranch := cicdOrch.Branch()
|
||||
if cicdBranch != "n/a" {
|
||||
branch = cicdBranch
|
||||
log.Entry().Infof("CxOne scan branch was automatically set to : %v", branch)
|
||||
} else {
|
||||
log.Entry().Info("Could not retrieve branch name from orchestrator")
|
||||
}
|
||||
}
|
||||
if len(c.config.PullRequestName) > 0 {
|
||||
branch = fmt.Sprintf("%v-%v", c.config.PullRequestName, c.config.Branch)
|
||||
branch = fmt.Sprintf("%v-%v", c.config.PullRequestName, branch)
|
||||
} else if cicdOrch.IsPullRequest() && cicdOrch.PullRequestConfig().Branch != "n/a" {
|
||||
branch = fmt.Sprintf("PR%v-%v", cicdOrch.PullRequestConfig().Key, cicdOrch.PullRequestConfig().Branch)
|
||||
}
|
||||
|
||||
baseBranch := cicdOrch.PullRequestConfig().Base
|
||||
isPR := cicdOrch.IsPullRequest()
|
||||
log.Entry().Debugf("CxOne scan branch was automatically set to : %v", branch)
|
||||
return branch, isPR, baseBranch
|
||||
}
|
||||
|
||||
func (c *checkmarxOneExecuteScanHelper) CreateScanRequest(incremental bool, uploadLink string, baseBranch string) (*checkmarxOne.Scan, error) {
|
||||
sastConfigString := ""
|
||||
sastConfig := checkmarxOne.ScanConfiguration{}
|
||||
sastConfig.ScanType = "sast"
|
||||
|
||||
sastConfig.Values = make(map[string]string, 0)
|
||||
sastConfig.Values["incremental"] = strconv.FormatBool(incremental)
|
||||
sastConfig.Values["presetName"] = c.config.Preset // always set, either coming from config or coming from Cx1 configuration
|
||||
if incremental && len(baseBranch) > 0 { // base the incremental scan on the specified base branch
|
||||
sastConfig.Values["baseBranch"] = baseBranch
|
||||
sastConfigString = fmt.Sprintf("baseBranch: %v, ", baseBranch)
|
||||
}
|
||||
sastConfigString = fmt.Sprintf("%vincremental %v, preset %v", sastConfigString, strconv.FormatBool(incremental), c.config.Preset)
|
||||
|
||||
if len(c.config.LanguageMode) > 0 {
|
||||
sastConfig.Values["languageMode"] = c.config.LanguageMode
|
||||
sastConfigString = sastConfigString + fmt.Sprintf(", languageMode %v", c.config.LanguageMode)
|
||||
}
|
||||
|
||||
branch, _, _ := c.GetScanBranch()
|
||||
|
||||
sastConfigString = fmt.Sprintf("Cx1 Branch name %v, ", branch) + sastConfigString
|
||||
|
||||
log.Entry().Infof("Will run a scan with the following configuration: %v", sastConfigString)
|
||||
@@ -1028,6 +1075,7 @@ func (c *checkmarxOneExecuteScanHelper) getDetailedResults(scan *checkmarxOne.Sc
|
||||
|
||||
func (c *checkmarxOneExecuteScanHelper) zipWorkspaceFiles(filterPattern string, utils checkmarxOneExecuteScanUtils) (*os.File, error) {
|
||||
zipFileName := filepath.Join(utils.GetWorkspace(), "workspace.zip")
|
||||
log.Entry().Infof("Zipping files using filter: %v", filterPattern)
|
||||
patterns := piperutils.Trim(strings.Split(filterPattern, ","))
|
||||
sort.Strings(patterns)
|
||||
zipFile, err := os.Create(zipFileName)
|
||||
|
@@ -70,11 +70,11 @@ func (sys *checkmarxOneSystemMock) GetScanWorkflow(scanID string) ([]checkmarxOn
|
||||
return []checkmarxOne.WorkflowLog{}, nil
|
||||
}
|
||||
|
||||
func (sys *checkmarxOneSystemMock) GetLastScans(projectID string, limit int) ([]checkmarxOne.Scan, error) {
|
||||
func (sys *checkmarxOneSystemMock) GetLastScans(projectID, branch string, limit int) ([]checkmarxOne.Scan, error) {
|
||||
return []checkmarxOne.Scan{}, nil
|
||||
}
|
||||
|
||||
func (sys *checkmarxOneSystemMock) GetLastScansByStatus(projectID string, limit int, status []string) ([]checkmarxOne.Scan, error) {
|
||||
func (sys *checkmarxOneSystemMock) GetLastScansByStatus(projectID, branch string, limit int, status []string) ([]checkmarxOne.Scan, error) {
|
||||
return []checkmarxOne.Scan{}, nil
|
||||
}
|
||||
|
||||
|
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
@@ -136,11 +137,38 @@ func printCodeqlImageVersion() {
|
||||
}
|
||||
}
|
||||
|
||||
func prepareCodeQLConfigFile(config *codeqlExecuteScanOptions) error {
|
||||
pathsToScan := codeql.ParsePaths(config.Paths)
|
||||
pathsToIgnore := codeql.ParsePaths(config.PathsIgnore)
|
||||
|
||||
codeQLExecName := "codeql"
|
||||
codeQLPath, err := codeql.Which(codeQLExecName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not locate codeql executable %w", err)
|
||||
}
|
||||
location, fileName := path.Split(codeQLPath)
|
||||
if fileName != codeQLExecName {
|
||||
return fmt.Errorf("could not find codeql executable in path: %s", codeQLPath)
|
||||
}
|
||||
defaultConfigLocation := path.Join(location, "default-codeql-config.yml")
|
||||
err = codeql.AppendCodeQLPaths(defaultConfigLocation, pathsToScan, pathsToIgnore)
|
||||
if err != nil {
|
||||
return fmt.Errorf("append paths and paths ignore to the default config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, utils codeqlExecuteScanUtils, influx *codeqlExecuteScanInflux) ([]piperutils.Path, error) {
|
||||
printCodeqlImageVersion()
|
||||
|
||||
var reports []piperutils.Path
|
||||
|
||||
err := prepareCodeQLConfigFile(config)
|
||||
if err != nil {
|
||||
log.Entry().WithError(err).Error("failed to prepare codeql config file")
|
||||
return reports, err
|
||||
}
|
||||
|
||||
dbCreateCustomFlags := codeql.ParseCustomFlags(config.DatabaseCreateFlags)
|
||||
isMultiLang, err := runDatabaseCreate(config, dbCreateCustomFlags, utils)
|
||||
if err != nil {
|
||||
|
@@ -48,6 +48,8 @@ type codeqlExecuteScanOptions struct {
|
||||
DatabaseAnalyzeFlags string `json:"databaseAnalyzeFlags,omitempty"`
|
||||
CustomCommand string `json:"customCommand,omitempty"`
|
||||
TransformQuerySuite string `json:"transformQuerySuite,omitempty"`
|
||||
Paths string `json:"paths,omitempty"`
|
||||
PathsIgnore string `json:"pathsIgnore,omitempty"`
|
||||
}
|
||||
|
||||
type codeqlExecuteScanInflux struct {
|
||||
@@ -306,6 +308,8 @@ func addCodeqlExecuteScanFlags(cmd *cobra.Command, stepConfig *codeqlExecuteScan
|
||||
cmd.Flags().StringVar(&stepConfig.DatabaseAnalyzeFlags, "databaseAnalyzeFlags", os.Getenv("PIPER_databaseAnalyzeFlags"), "A space-separated string of flags for the 'codeql database analyze' command.")
|
||||
cmd.Flags().StringVar(&stepConfig.CustomCommand, "customCommand", os.Getenv("PIPER_customCommand"), "A custom user-defined command to run between codeql analysis and results upload.")
|
||||
cmd.Flags().StringVar(&stepConfig.TransformQuerySuite, "transformQuerySuite", os.Getenv("PIPER_transformQuerySuite"), "A transform string that will be applied to the querySuite using the sed command.")
|
||||
cmd.Flags().StringVar(&stepConfig.Paths, "paths", os.Getenv("PIPER_paths"), "List of file or directory patterns to include.\nEach entry must be on its own line, e.g.:\n src/**\n lib/**\nNote: This parameter is only applicable for interpreted languages.\n")
|
||||
cmd.Flags().StringVar(&stepConfig.PathsIgnore, "pathsIgnore", os.Getenv("PIPER_pathsIgnore"), "List of file or directory patterns to ignore.\nEach entry must be on its own line, e.g.:\n **/*.md\n docs/**\nNote: This parameter is only applicable for interpreted languages.\n")
|
||||
|
||||
cmd.MarkFlagRequired("buildTool")
|
||||
}
|
||||
@@ -580,6 +584,24 @@ func codeqlExecuteScanMetadata() config.StepData {
|
||||
Aliases: []config.Alias{},
|
||||
Default: os.Getenv("PIPER_transformQuerySuite"),
|
||||
},
|
||||
{
|
||||
Name: "paths",
|
||||
ResourceRef: []config.ResourceReference{},
|
||||
Scope: []string{"STEPS", "STAGES", "PARAMETERS"},
|
||||
Type: "string",
|
||||
Mandatory: false,
|
||||
Aliases: []config.Alias{},
|
||||
Default: os.Getenv("PIPER_paths"),
|
||||
},
|
||||
{
|
||||
Name: "pathsIgnore",
|
||||
ResourceRef: []config.ResourceReference{},
|
||||
Scope: []string{"STEPS", "STAGES", "PARAMETERS"},
|
||||
Type: "string",
|
||||
Mandatory: false,
|
||||
Aliases: []config.Alias{},
|
||||
Default: os.Getenv("PIPER_pathsIgnore"),
|
||||
},
|
||||
},
|
||||
},
|
||||
Containers: []config.Container{
|
||||
|
@@ -7,11 +7,15 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/SAP/jenkins-library/pkg/codeql"
|
||||
"github.com/SAP/jenkins-library/pkg/mock"
|
||||
@@ -1077,3 +1081,148 @@ func TestRunCustomCommand(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func Test_prepareCodeQLConfigFile(t *testing.T) {
|
||||
t.Run("creates_or_updates_default_config_next_to_codeql_binary", func(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("requires POSIX exec bits for the fake binary")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
|
||||
codeqlBin := filepath.Join(dir, "codeql")
|
||||
writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n")
|
||||
|
||||
origPath := os.Getenv("PATH")
|
||||
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
|
||||
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
|
||||
|
||||
opts := &codeqlExecuteScanOptions{
|
||||
Paths: " src \nlib/utils\n",
|
||||
PathsIgnore: "vendor\n**/*.gen.go\n",
|
||||
}
|
||||
|
||||
err := prepareCodeQLConfigFile(opts)
|
||||
require.NoError(t, err)
|
||||
|
||||
qlPath, err := codeql.Which("codeql")
|
||||
require.NoError(t, err)
|
||||
|
||||
loc, found := strings.CutSuffix(qlPath, "codeql")
|
||||
require.True(t, found)
|
||||
|
||||
cfgPath := path.Join(loc, "default-codeql-config.yml")
|
||||
b, err := os.ReadFile(cfgPath)
|
||||
require.NoError(t, err)
|
||||
out := normalizeNL(string(b))
|
||||
|
||||
// Assert it contains the expected lists (don’t depend on ordering)
|
||||
assert.Contains(t, out, "paths:")
|
||||
assert.Contains(t, out, "- src")
|
||||
assert.Contains(t, out, "- lib/utils")
|
||||
|
||||
assert.Contains(t, out, "paths-ignore:")
|
||||
assert.Contains(t, out, "- vendor")
|
||||
// yaml.v3 single-quotes strings with '*' etc.
|
||||
assert.Contains(t, out, "- '**/*.gen.go'")
|
||||
})
|
||||
|
||||
t.Run("when_codeql_binary_missing_returns_error", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
origPath := os.Getenv("PATH")
|
||||
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
|
||||
|
||||
// PATH without codeql
|
||||
require.NoError(t, os.Setenv("PATH", dir))
|
||||
|
||||
opts := &codeqlExecuteScanOptions{
|
||||
Paths: "a",
|
||||
PathsIgnore: "b",
|
||||
}
|
||||
err := prepareCodeQLConfigFile(opts)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "could not locate codeql executable")
|
||||
})
|
||||
|
||||
t.Run("propagates_append_error_when_default_config_path_is_a_directory", func(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("requires POSIX exec bits and reliable directory perms")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
|
||||
// Create fake codeql binary
|
||||
codeqlBin := filepath.Join(dir, "codeql")
|
||||
writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n")
|
||||
|
||||
defaultCfgDir := filepath.Join(dir, "default-codeql-config.yml")
|
||||
require.NoError(t, os.Mkdir(defaultCfgDir, 0o755))
|
||||
|
||||
origPath := os.Getenv("PATH")
|
||||
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
|
||||
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
|
||||
|
||||
opts := &codeqlExecuteScanOptions{
|
||||
Paths: "x",
|
||||
PathsIgnore: "y",
|
||||
}
|
||||
err := prepareCodeQLConfigFile(opts)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "append paths and paths ignore to the default config")
|
||||
})
|
||||
|
||||
t.Run("no_changes_when_both_paths_empty_but_still_needs_binary", func(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("requires POSIX exec bits for the fake binary")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
|
||||
codeqlBin := filepath.Join(dir, "codeql")
|
||||
writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n")
|
||||
origPath := os.Getenv("PATH")
|
||||
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
|
||||
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
|
||||
|
||||
cfgFile := filepath.Join(dir, "default-codeql-config.yml")
|
||||
writeCodeQLFile(t, cfgFile, "")
|
||||
|
||||
opts := &codeqlExecuteScanOptions{
|
||||
Paths: "",
|
||||
PathsIgnore: "",
|
||||
}
|
||||
|
||||
before := readCodeQLFile(t, cfgFile)
|
||||
time.Sleep(10 * time.Millisecond) // avoid flakiness on very fast FS
|
||||
|
||||
err := prepareCodeQLConfigFile(opts)
|
||||
require.NoError(t, err)
|
||||
|
||||
after := readCodeQLFile(t, cfgFile)
|
||||
assert.Equal(t, before, after, "file should remain unchanged when nothing to write")
|
||||
})
|
||||
}
|
||||
|
||||
func writeExec(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
require.NoError(t, os.WriteFile(path, []byte(content), 0o755))
|
||||
require.NoError(t, os.Chmod(path, 0o755))
|
||||
info, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
require.False(t, info.IsDir())
|
||||
require.NotZero(t, info.Mode()&0o111, "should be executable")
|
||||
}
|
||||
|
||||
func writeCodeQLFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
|
||||
}
|
||||
|
||||
func readCodeQLFile(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func normalizeNL(s string) string {
|
||||
return strings.ReplaceAll(s, "\r\n", "\n")
|
||||
}
|
||||
|
@@ -304,8 +304,8 @@ type System interface {
|
||||
GetScanSummary(scanID string) (ScanSummary, error)
|
||||
GetResultsPredicates(SimilarityID int64, ProjectID string) ([]ResultsPredicates, error)
|
||||
GetScanWorkflow(scanID string) ([]WorkflowLog, error)
|
||||
GetLastScans(projectID string, limit int) ([]Scan, error)
|
||||
GetLastScansByStatus(projectID string, limit int, status []string) ([]Scan, error)
|
||||
GetLastScans(projectID, branch string, limit int) ([]Scan, error)
|
||||
GetLastScansByStatus(projectID, branch string, limit int, status []string) ([]Scan, error)
|
||||
|
||||
ScanProject(projectID, sourceUrl, branch, scanType string, settings []ScanConfiguration, tags map[string]string) (Scan, error)
|
||||
ScanProjectZip(projectID, sourceUrl, branch string, settings []ScanConfiguration, tags map[string]string) (Scan, error)
|
||||
@@ -1142,7 +1142,7 @@ func (sys *SystemInstance) GetScanWorkflow(scanID string) ([]WorkflowLog, error)
|
||||
return workflow, nil
|
||||
}
|
||||
|
||||
func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, error) {
|
||||
func (sys *SystemInstance) GetLastScans(projectID, branch string, limit int) ([]Scan, error) {
|
||||
var scanResponse struct {
|
||||
TotalCount uint64
|
||||
FilteredTotalCount uint64
|
||||
@@ -1163,6 +1163,7 @@ func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, er
|
||||
"offset": {fmt.Sprintf("%v", 0)},
|
||||
"limit": {fmt.Sprintf("%v", limit)},
|
||||
"sort": {sortStr},
|
||||
"branches": []string{branch},
|
||||
}
|
||||
|
||||
header := http.Header{}
|
||||
@@ -1177,7 +1178,7 @@ func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, er
|
||||
return scanResponse.Scans, err
|
||||
}
|
||||
|
||||
func (sys *SystemInstance) GetLastScansByStatus(projectID string, limit int, status []string) ([]Scan, error) {
|
||||
func (sys *SystemInstance) GetLastScansByStatus(projectID, branch string, limit int, status []string) ([]Scan, error) {
|
||||
var scanResponse struct {
|
||||
TotalCount uint64
|
||||
FilteredTotalCount uint64
|
||||
@@ -1199,6 +1200,7 @@ func (sys *SystemInstance) GetLastScansByStatus(projectID string, limit int, sta
|
||||
"limit": {fmt.Sprintf("%d", limit)},
|
||||
"sort": {sortStr},
|
||||
"statuses": status,
|
||||
"branches": []string{branch},
|
||||
}
|
||||
|
||||
data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/scans/?%v", body.Encode()), nil, nil, []int{})
|
||||
|
105
pkg/codeql/path.go
Normal file
105
pkg/codeql/path.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package codeql
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// AppendCodeQLPaths updates the CodeQL config YAML with new paths/paths-ignore.
|
||||
func AppendCodeQLPaths(cfgPath string, scanPaths, ignorePaths []string) error {
|
||||
if len(scanPaths) == 0 && len(ignorePaths) == 0 {
|
||||
// if both paths are empty - do not touch anything
|
||||
return nil
|
||||
}
|
||||
var cfg map[string]any
|
||||
|
||||
data, err := os.ReadFile(cfgPath)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// file exists; unmarshal if non-empty
|
||||
if len(data) > 0 {
|
||||
if err = yaml.Unmarshal(data, &cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if cfg == nil {
|
||||
// start from empty config if file doesn't exist or empty
|
||||
cfg = make(map[string]any)
|
||||
}
|
||||
|
||||
if len(scanPaths) != 0 {
|
||||
cfg["paths"] = scanPaths
|
||||
}
|
||||
if len(ignorePaths) != 0 {
|
||||
cfg["paths-ignore"] = ignorePaths
|
||||
}
|
||||
|
||||
out, err := yaml.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(cfgPath, out, 0o644)
|
||||
}
|
||||
|
||||
func ParsePaths(pathsStr string) []string {
|
||||
var paths []string
|
||||
patterns := strings.Split(pathsStr, "\n")
|
||||
for _, p := range patterns {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
paths = append(paths, p)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
// Which finds the first executable in PATH and resolves symlinks.
|
||||
func Which(name string) (string, error) {
|
||||
// If the name already has a path separator, check it directly.
|
||||
if strings.ContainsRune(name, filepath.Separator) {
|
||||
if isExecutable(name) {
|
||||
return resolve(name)
|
||||
}
|
||||
return "", os.ErrNotExist
|
||||
}
|
||||
|
||||
// Search in PATH
|
||||
for _, dir := range filepath.SplitList(os.Getenv("PATH")) {
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
candidate := filepath.Join(dir, name)
|
||||
if isExecutable(candidate) {
|
||||
return resolve(candidate)
|
||||
}
|
||||
}
|
||||
return "", os.ErrNotExist
|
||||
}
|
||||
|
||||
func isExecutable(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if info.IsDir() {
|
||||
return false
|
||||
}
|
||||
// POSIX: check execute bit
|
||||
return info.Mode()&0o111 != 0
|
||||
}
|
||||
|
||||
func resolve(path string) (string, error) {
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.EvalSymlinks(abs)
|
||||
}
|
326
pkg/codeql/path_test.go
Normal file
326
pkg/codeql/path_test.go
Normal file
@@ -0,0 +1,326 @@
|
||||
package codeql
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAppendCodeQLPaths(t *testing.T) {
|
||||
t.Run("AppendCodeQLPaths", func(t *testing.T) {
|
||||
t.Run("creates_file_if_not_exists_with_both_keys", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "codeql.yml")
|
||||
|
||||
err := AppendCodeQLPaths(cfgPath,
|
||||
[]string{"src", "lib/utils"},
|
||||
[]string{"vendor", "**/*.gen.go"},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
out := readFile(t, cfgPath)
|
||||
assert.Contains(t, out, "paths:")
|
||||
assert.Contains(t, out, "- src")
|
||||
assert.Contains(t, out, "- lib/utils")
|
||||
|
||||
assert.Contains(t, out, "paths-ignore:")
|
||||
assert.Contains(t, out, "- vendor")
|
||||
assert.Contains(t, out, "- '**/*.gen.go'") // YAML encoder uses single quotes for '*'
|
||||
})
|
||||
|
||||
t.Run("creates_file_if_not_exists_with_only_paths", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "codeql.yml")
|
||||
|
||||
err := AppendCodeQLPaths(cfgPath, []string{"a", "b"}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
out := readFile(t, cfgPath)
|
||||
assert.Contains(t, out, "paths:")
|
||||
assert.Contains(t, out, "- a")
|
||||
assert.Contains(t, out, "- b")
|
||||
assert.NotContains(t, out, "paths-ignore:")
|
||||
})
|
||||
|
||||
t.Run("creates_file_if_not_exists_with_only_paths_ignore", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "codeql.yml")
|
||||
|
||||
err := AppendCodeQLPaths(cfgPath, nil, []string{"vendor"})
|
||||
require.NoError(t, err)
|
||||
|
||||
out := readFile(t, cfgPath)
|
||||
assert.NotContains(t, out, "paths:\n")
|
||||
assert.Contains(t, out, "paths-ignore:")
|
||||
assert.Contains(t, out, "- vendor")
|
||||
})
|
||||
|
||||
t.Run("appends_to_empty_yaml_creates_paths_and_paths_ignore", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "codeql.yml")
|
||||
writeFile(t, cfgPath, "")
|
||||
|
||||
err := AppendCodeQLPaths(cfgPath,
|
||||
[]string{"src", "lib/utils"},
|
||||
[]string{"vendor", "**/*.gen.go"},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
out := readFile(t, cfgPath)
|
||||
|
||||
assert.Contains(t, out, "paths:")
|
||||
assert.Contains(t, out, "- src")
|
||||
assert.Contains(t, out, "- lib/utils")
|
||||
|
||||
assert.Contains(t, out, "paths-ignore:")
|
||||
assert.Contains(t, out, "- vendor")
|
||||
assert.Contains(t, out, "- '**/*.gen.go'")
|
||||
})
|
||||
|
||||
t.Run("overwrites_on_type_mismatch_and_preserves_other_keys", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "codeql.yml")
|
||||
// Intentionally set scalar for paths to force overwrite; and keep an existing ignore.
|
||||
initial := "paths: foo\npaths-ignore:\n - old-ignore\n"
|
||||
writeFile(t, cfgPath, initial)
|
||||
|
||||
err := AppendCodeQLPaths(cfgPath, []string{"new-path"}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
out := readFile(t, cfgPath)
|
||||
|
||||
// 'paths' should now be a sequence containing only the new value
|
||||
assert.Contains(t, out, "paths:")
|
||||
assert.Contains(t, out, "- new-path")
|
||||
assert.NotContains(t, out, "foo")
|
||||
|
||||
// paths-ignore should remain present (we didn't provide a new ignore list)
|
||||
assert.Contains(t, out, "paths-ignore:")
|
||||
assert.Contains(t, out, "- old-ignore")
|
||||
})
|
||||
|
||||
t.Run("updates_only_paths_when_ignore_not_provided", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "codeql.yml")
|
||||
initial := "paths:\n - old\npaths-ignore:\n - keep-ignore\n"
|
||||
writeFile(t, cfgPath, initial)
|
||||
|
||||
err := AppendCodeQLPaths(cfgPath, []string{"new-a", "new-b"}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
out := readFile(t, cfgPath)
|
||||
assert.Contains(t, out, "paths:")
|
||||
assert.Contains(t, out, "- new-a")
|
||||
assert.Contains(t, out, "- new-b")
|
||||
assert.NotContains(t, out, "- old")
|
||||
|
||||
assert.Contains(t, out, "paths-ignore:")
|
||||
assert.Contains(t, out, "- keep-ignore")
|
||||
})
|
||||
|
||||
t.Run("updates_only_paths_ignore_when_paths_not_provided", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "codeql.yml")
|
||||
initial := "paths:\n - keep\npaths-ignore:\n - old-ignore\n"
|
||||
writeFile(t, cfgPath, initial)
|
||||
|
||||
err := AppendCodeQLPaths(cfgPath, nil, []string{"new-ignore-1", "**/*.gen.go"})
|
||||
require.NoError(t, err)
|
||||
|
||||
out := readFile(t, cfgPath)
|
||||
assert.Contains(t, out, "paths:")
|
||||
assert.Contains(t, out, "- keep")
|
||||
|
||||
assert.Contains(t, out, "paths-ignore:")
|
||||
assert.Contains(t, out, "- new-ignore-1")
|
||||
assert.Contains(t, out, "- '**/*.gen.go'")
|
||||
assert.NotContains(t, out, "- old-ignore")
|
||||
})
|
||||
|
||||
t.Run("no_op_when_no_values_on_existing_file", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "codeql.yml")
|
||||
initial := "paths:\n - keep\npaths-ignore:\n - keep-ignore\n"
|
||||
writeFile(t, cfgPath, initial)
|
||||
before := readFile(t, cfgPath)
|
||||
|
||||
// Touch mtime differences robustly across filesystems.
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
err := AppendCodeQLPaths(cfgPath, nil, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
after := readFile(t, cfgPath)
|
||||
assert.Equal(t, before, after, "file content should be unchanged for no-op with nil slices")
|
||||
})
|
||||
|
||||
t.Run("no_op_when_no_values_and_file_missing_does_not_create_file", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "codeql.yml")
|
||||
|
||||
err := AppendCodeQLPaths(cfgPath, []string{}, []string{})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, statErr := os.Stat(cfgPath)
|
||||
require.True(t, os.IsNotExist(statErr), "file should not be created when both inputs are empty")
|
||||
})
|
||||
|
||||
t.Run("works_with_existing_empty_file", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "codeql.yml")
|
||||
writeFile(t, cfgPath, "")
|
||||
|
||||
err := AppendCodeQLPaths(cfgPath, []string{"p1"}, []string{"i1"})
|
||||
require.NoError(t, err)
|
||||
|
||||
out := readFile(t, cfgPath)
|
||||
assert.Contains(t, out, "paths:")
|
||||
assert.Contains(t, out, "- p1")
|
||||
assert.Contains(t, out, "paths-ignore:")
|
||||
assert.Contains(t, out, "- i1")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestParsePaths(t *testing.T) {
|
||||
t.Run("ParsePaths", func(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
out []string
|
||||
}{
|
||||
{
|
||||
name: "trims_and_skips_blanks",
|
||||
in: " src \n\nlib\n \nvendor ",
|
||||
out: []string{"src", "lib", "vendor"},
|
||||
},
|
||||
{
|
||||
name: "handles_single_line",
|
||||
in: "only/one",
|
||||
out: []string{"only/one"},
|
||||
},
|
||||
{
|
||||
name: "keeps_globs_and_spaces_inside_line",
|
||||
in: "**/*.go\npath with space/\n./rel",
|
||||
out: []string{"**/*.go", "path with space/", "./rel"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := ParsePaths(tc.in)
|
||||
assert.Equal(t, tc.out, got)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestWhich(t *testing.T) {
|
||||
t.Run("Which", func(t *testing.T) {
|
||||
// Save and restore PATH for isolation.
|
||||
origPath := os.Getenv("PATH")
|
||||
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
|
||||
|
||||
t.Run("nonexistent_returns_ErrNotExist", func(t *testing.T) {
|
||||
_, err := Which("definitely-not-a-real-binary-name")
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
})
|
||||
|
||||
t.Run("finds_executable_in_PATH", func(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("executable bit semantics differ on Windows; test skipped")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
bin := filepath.Join(dir, "mytool")
|
||||
writeExec(t, bin, "#!/bin/sh\necho ok\n")
|
||||
|
||||
// Put our temp dir first on PATH
|
||||
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
|
||||
|
||||
found, err := Which("mytool")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Compare against the resolved path (handles /var -> /private/var on macOS)
|
||||
expected, err := resolve(bin)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expected, found)
|
||||
})
|
||||
|
||||
t.Run("resolves_symlinks", func(t *testing.T) {
|
||||
// Symlink behavior varies on Windows (needs admin/developer mode).
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("symlink creation is restricted on Windows; test skipped")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
target := filepath.Join(dir, "realtool")
|
||||
link := filepath.Join(dir, "alias")
|
||||
|
||||
writeExec(t, target, "#!/bin/sh\necho real\n")
|
||||
err := os.Symlink(target, link)
|
||||
if err != nil {
|
||||
t.Skipf("symlinks not supported in this environment: %v", err)
|
||||
}
|
||||
|
||||
require.NoError(t, os.Setenv("PATH", dir))
|
||||
|
||||
found, err := Which("alias")
|
||||
require.NoError(t, err)
|
||||
|
||||
expected, err := resolve(target)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expected, found, "Which should resolve through the symlink to the real path")
|
||||
})
|
||||
|
||||
t.Run("direct_path_with_separator", func(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("executable bit semantics differ on Windows; test skipped")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
bin := filepath.Join(dir, "runme")
|
||||
writeExec(t, bin, "#!/bin/sh\necho hi\n")
|
||||
|
||||
found, err := Which(bin)
|
||||
require.NoError(t, err)
|
||||
|
||||
expected, err := resolve(bin)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expected, found)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
err := os.WriteFile(path, []byte(content), 0o644)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func writeExec(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
err := os.WriteFile(path, []byte(content), 0o755)
|
||||
require.NoError(t, err)
|
||||
|
||||
// On some systems, writing then chmod is safer/more explicit.
|
||||
err = os.Chmod(path, 0o755)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Sanity: ensure it's not a directory and has any execute bit set (for non-Windows cases we test)
|
||||
info, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
require.False(t, info.IsDir())
|
||||
require.True(t, info.Mode()&0o111 != 0)
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
// Normalize Windows newlines just in case, so substring checks are reliable.
|
||||
return strings.ReplaceAll(string(b), "\r\n", "\n")
|
||||
}
|
@@ -247,6 +247,31 @@ spec:
|
||||
- STEPS
|
||||
- STAGES
|
||||
- PARAMETERS
|
||||
- name: paths
|
||||
type: string
|
||||
description: |
|
||||
List of file or directory patterns to include.
|
||||
Each entry must be on its own line, e.g.:
|
||||
src/**
|
||||
lib/**
|
||||
Note: This parameter is only applicable for interpreted languages.
|
||||
scope:
|
||||
- STEPS
|
||||
- STAGES
|
||||
- PARAMETERS
|
||||
- name: pathsIgnore
|
||||
type: string
|
||||
pathsIgnore:
|
||||
description: |
|
||||
List of file or directory patterns to ignore.
|
||||
Each entry must be on its own line, e.g.:
|
||||
**/*.md
|
||||
docs/**
|
||||
Note: This parameter is only applicable for interpreted languages.
|
||||
scope:
|
||||
- STEPS
|
||||
- STAGES
|
||||
- PARAMETERS
|
||||
containers:
|
||||
- image: ""
|
||||
outputs:
|
||||
|
Reference in New Issue
Block a user