1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-09-16 09:26:22 +02:00

Merge branch 'master' into phgermanov/error-patterns-top-10

This commit is contained in:
Googlom
2025-09-15 11:03:29 +03:00
committed by GitHub
10 changed files with 761 additions and 28 deletions

28
.github/renovate.json vendored
View File

@@ -104,6 +104,34 @@
"depNameTemplate": "CycloneDX/{{{depName}}}",
"extractVersionTemplate": "v(?<version>.*)",
"autoReplaceStringTemplate": "\"github.com/{{{depName}}}/cmd/cyclonedx-gomod@v{{{newValue}}}\""
},
{
"customType": "regex",
"matchStringsStrategy": "any",
"managerFilePatterns": [
"resources/metadata/pythonBuild.yaml"
],
"matchStrings": [
"image: (?<depName>python):(?<currentValue>.*)"
],
"depTypeTemplate": "dependencies",
"datasourceTemplate": "docker",
"extractVersionTemplate": "(?<version>.*)",
"autoReplaceStringTemplate": "image: {{depName}}:{{newValue}}"
},
{
"customType": "regex",
"matchStringsStrategy": "any",
"managerFilePatterns": [
"cmd/pythonBuild_generated.go"
],
"matchStrings": [
"Image: \"(?<depName>python):(?<currentValue>.*)\""
],
"depTypeTemplate": "dependencies",
"datasourceTemplate": "docker",
"extractVersionTemplate": "(?<version>.*)",
"autoReplaceStringTemplate": "Image: \"{{depName}}:{{newValue}}\""
}
],
"postUpdateOptions": [

View File

@@ -128,7 +128,8 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS
}
}
scans, err := cx1sh.GetLastScans(10)
branch, isPR, baseBranch := cx1sh.GetScanBranch()
scans, err := cx1sh.GetLastScans(10, branch)
if err != nil {
log.Entry().WithError(err).Warnf("failed to get last 10 scans")
}
@@ -158,7 +159,7 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS
}
if config.Incremental {
log.Entry().Warnf("If you change your file filter pattern it is recommended to run a Full scan instead of an incremental, to ensure full code coverage.")
log.Entry().Info("If you change your file filter pattern it is recommended to run a Full scan instead of an incremental, to ensure full code coverage.")
}
zipFile, err := cx1sh.ZipFiles()
@@ -172,7 +173,40 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS
}
// TODO : The step structure should allow to enable different scanners: SAST, KICKS, SCA
scan, err := cx1sh.CreateScanRequest(incremental, uploadLink)
var scan *checkmarxOne.Scan
// We search for a full scan in history:
fullScanExists := false
for _, histScan := range scans {
isIncr, _ := histScan.IsIncremental()
if isIncr == false {
fullScanExists = true
break
}
}
// user requested an incremental scan on a branch, and the project has a Primary Branch set, not in PR context and no full scan on the branch
if config.Incremental && !isPR && !fullScanExists && cx1sh.Project.MainBranch != "" && cx1sh.Project.MainBranch != branch {
scansMainBranch, err := cx1sh.GetLastScans(10, cx1sh.Project.MainBranch)
if err != nil {
return fmt.Errorf("failed to determine incremental or full scan configuration: %s", err)
}
// We check if the main branch is eligible for an incremental scan
incrementalMainBranch, err := cx1sh.IncrementalOrFull(scansMainBranch)
log.Entry().Debugf("Main branch %v incremental scan eligibility: %t", cx1sh.Project.MainBranch, incrementalMainBranch)
scan, err = cx1sh.CreateScanRequest(incrementalMainBranch, uploadLink, cx1sh.Project.MainBranch) // this will create a full scan on the current branch if the main branch is not eligible for an incremental scan
} else if config.Incremental && isPR && len(baseBranch) > 0 && baseBranch != "n/a" { // running in a PR context, and we have a base branch for the incremental scan
// in a PR context we always want to do an incremental scan
// The scan will be based on the PR's target branch (baseBranch) if there is no full scan on the PR branch
if fullScanExists {
log.Entry().Debugf("A full scan exists on the PR branch %v, so the incremental scan will be based on it", branch)
scan, err = cx1sh.CreateScanRequest(true, uploadLink, "")
} else {
log.Entry().Debugf("There is no full scan on the PR branch %v, so the incremental scan will be based on branch %v", branch, baseBranch)
scan, err = cx1sh.CreateScanRequest(true, uploadLink, baseBranch)
}
} else {
scan, err = cx1sh.CreateScanRequest(incremental, uploadLink, "")
}
if err != nil {
return fmt.Errorf("failed to create scan: %s", err)
}
@@ -388,8 +422,8 @@ func (c *checkmarxOneExecuteScanHelper) SetProjectPreset() error {
return nil
}
func (c *checkmarxOneExecuteScanHelper) GetLastScans(count int) ([]checkmarxOne.Scan, error) {
scans, err := c.sys.GetLastScansByStatus(c.Project.ProjectID, count, []string{"Completed"})
func (c *checkmarxOneExecuteScanHelper) GetLastScans(count int, branch string) ([]checkmarxOne.Scan, error) {
scans, err := c.sys.GetLastScansByStatus(c.Project.ProjectID, branch, count, []string{"Completed"})
if err != nil {
return []checkmarxOne.Scan{}, fmt.Errorf("Failed to get last %d Completed scans for project %v: %s", count, c.Project.ProjectID, err)
}
@@ -437,23 +471,10 @@ func (c *checkmarxOneExecuteScanHelper) UploadScanContent(zipFile *os.File) (str
return uploadUri, nil
}
func (c *checkmarxOneExecuteScanHelper) CreateScanRequest(incremental bool, uploadLink string) (*checkmarxOne.Scan, error) {
sastConfig := checkmarxOne.ScanConfiguration{}
sastConfig.ScanType = "sast"
sastConfig.Values = make(map[string]string, 0)
sastConfig.Values["incremental"] = strconv.FormatBool(incremental)
sastConfig.Values["presetName"] = c.config.Preset // always set, either coming from config or coming from Cx1 configuration
sastConfigString := fmt.Sprintf("incremental %v, preset %v", strconv.FormatBool(incremental), c.config.Preset)
if len(c.config.LanguageMode) > 0 {
sastConfig.Values["languageMode"] = c.config.LanguageMode
sastConfigString = sastConfigString + fmt.Sprintf(", languageMode %v", c.config.LanguageMode)
}
func (c *checkmarxOneExecuteScanHelper) GetScanBranch() (string, bool, string) {
branch := c.config.Branch
cicdOrch, err := orchestrator.GetOrchestratorConfigProvider(nil)
if err == nil {
if err != nil {
log.Entry().Warn("Could not identify orchestrator")
}
if len(branch) == 0 && len(c.config.GitBranch) > 0 && c.config.GitBranch != "n/a" {
@@ -462,17 +483,43 @@ func (c *checkmarxOneExecuteScanHelper) CreateScanRequest(incremental bool, uplo
cicdBranch := cicdOrch.Branch()
if cicdBranch != "n/a" {
branch = cicdBranch
log.Entry().Infof("CxOne scan branch was automatically set to : %v", branch)
} else {
log.Entry().Info("Could not retrieve branch name from orchestrator")
}
}
if len(c.config.PullRequestName) > 0 {
branch = fmt.Sprintf("%v-%v", c.config.PullRequestName, c.config.Branch)
branch = fmt.Sprintf("%v-%v", c.config.PullRequestName, branch)
} else if cicdOrch.IsPullRequest() && cicdOrch.PullRequestConfig().Branch != "n/a" {
branch = fmt.Sprintf("PR%v-%v", cicdOrch.PullRequestConfig().Key, cicdOrch.PullRequestConfig().Branch)
}
baseBranch := cicdOrch.PullRequestConfig().Base
isPR := cicdOrch.IsPullRequest()
log.Entry().Debugf("CxOne scan branch was automatically set to : %v", branch)
return branch, isPR, baseBranch
}
func (c *checkmarxOneExecuteScanHelper) CreateScanRequest(incremental bool, uploadLink string, baseBranch string) (*checkmarxOne.Scan, error) {
sastConfigString := ""
sastConfig := checkmarxOne.ScanConfiguration{}
sastConfig.ScanType = "sast"
sastConfig.Values = make(map[string]string, 0)
sastConfig.Values["incremental"] = strconv.FormatBool(incremental)
sastConfig.Values["presetName"] = c.config.Preset // always set, either coming from config or coming from Cx1 configuration
if incremental && len(baseBranch) > 0 { // base the incremental scan on the specified base branch
sastConfig.Values["baseBranch"] = baseBranch
sastConfigString = fmt.Sprintf("baseBranch: %v, ", baseBranch)
}
sastConfigString = fmt.Sprintf("%vincremental %v, preset %v", sastConfigString, strconv.FormatBool(incremental), c.config.Preset)
if len(c.config.LanguageMode) > 0 {
sastConfig.Values["languageMode"] = c.config.LanguageMode
sastConfigString = sastConfigString + fmt.Sprintf(", languageMode %v", c.config.LanguageMode)
}
branch, _, _ := c.GetScanBranch()
sastConfigString = fmt.Sprintf("Cx1 Branch name %v, ", branch) + sastConfigString
log.Entry().Infof("Will run a scan with the following configuration: %v", sastConfigString)
@@ -1028,6 +1075,7 @@ func (c *checkmarxOneExecuteScanHelper) getDetailedResults(scan *checkmarxOne.Sc
func (c *checkmarxOneExecuteScanHelper) zipWorkspaceFiles(filterPattern string, utils checkmarxOneExecuteScanUtils) (*os.File, error) {
zipFileName := filepath.Join(utils.GetWorkspace(), "workspace.zip")
log.Entry().Infof("Zipping files using filter: %v", filterPattern)
patterns := piperutils.Trim(strings.Split(filterPattern, ","))
sort.Strings(patterns)
zipFile, err := os.Create(zipFileName)

View File

@@ -70,11 +70,11 @@ func (sys *checkmarxOneSystemMock) GetScanWorkflow(scanID string) ([]checkmarxOn
return []checkmarxOne.WorkflowLog{}, nil
}
func (sys *checkmarxOneSystemMock) GetLastScans(projectID string, limit int) ([]checkmarxOne.Scan, error) {
func (sys *checkmarxOneSystemMock) GetLastScans(projectID, branch string, limit int) ([]checkmarxOne.Scan, error) {
return []checkmarxOne.Scan{}, nil
}
func (sys *checkmarxOneSystemMock) GetLastScansByStatus(projectID string, limit int, status []string) ([]checkmarxOne.Scan, error) {
func (sys *checkmarxOneSystemMock) GetLastScansByStatus(projectID, branch string, limit int, status []string) ([]checkmarxOne.Scan, error) {
return []checkmarxOne.Scan{}, nil
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"os"
"path"
"path/filepath"
"strings"
@@ -136,11 +137,38 @@ func printCodeqlImageVersion() {
}
}
func prepareCodeQLConfigFile(config *codeqlExecuteScanOptions) error {
pathsToScan := codeql.ParsePaths(config.Paths)
pathsToIgnore := codeql.ParsePaths(config.PathsIgnore)
codeQLExecName := "codeql"
codeQLPath, err := codeql.Which(codeQLExecName)
if err != nil {
return fmt.Errorf("could not locate codeql executable %w", err)
}
location, fileName := path.Split(codeQLPath)
if fileName != codeQLExecName {
return fmt.Errorf("could not find codeql executable in path: %s", codeQLPath)
}
defaultConfigLocation := path.Join(location, "default-codeql-config.yml")
err = codeql.AppendCodeQLPaths(defaultConfigLocation, pathsToScan, pathsToIgnore)
if err != nil {
return fmt.Errorf("append paths and paths ignore to the default config: %w", err)
}
return nil
}
func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, utils codeqlExecuteScanUtils, influx *codeqlExecuteScanInflux) ([]piperutils.Path, error) {
printCodeqlImageVersion()
var reports []piperutils.Path
err := prepareCodeQLConfigFile(config)
if err != nil {
log.Entry().WithError(err).Error("failed to prepare codeql config file")
return reports, err
}
dbCreateCustomFlags := codeql.ParseCustomFlags(config.DatabaseCreateFlags)
isMultiLang, err := runDatabaseCreate(config, dbCreateCustomFlags, utils)
if err != nil {

View File

@@ -48,6 +48,8 @@ type codeqlExecuteScanOptions struct {
DatabaseAnalyzeFlags string `json:"databaseAnalyzeFlags,omitempty"`
CustomCommand string `json:"customCommand,omitempty"`
TransformQuerySuite string `json:"transformQuerySuite,omitempty"`
Paths string `json:"paths,omitempty"`
PathsIgnore string `json:"pathsIgnore,omitempty"`
}
type codeqlExecuteScanInflux struct {
@@ -306,6 +308,8 @@ func addCodeqlExecuteScanFlags(cmd *cobra.Command, stepConfig *codeqlExecuteScan
cmd.Flags().StringVar(&stepConfig.DatabaseAnalyzeFlags, "databaseAnalyzeFlags", os.Getenv("PIPER_databaseAnalyzeFlags"), "A space-separated string of flags for the 'codeql database analyze' command.")
cmd.Flags().StringVar(&stepConfig.CustomCommand, "customCommand", os.Getenv("PIPER_customCommand"), "A custom user-defined command to run between codeql analysis and results upload.")
cmd.Flags().StringVar(&stepConfig.TransformQuerySuite, "transformQuerySuite", os.Getenv("PIPER_transformQuerySuite"), "A transform string that will be applied to the querySuite using the sed command.")
cmd.Flags().StringVar(&stepConfig.Paths, "paths", os.Getenv("PIPER_paths"), "List of file or directory patterns to include.\nEach entry must be on its own line, e.g.:\n src/**\n lib/**\nNote: This parameter is only applicable for interpreted languages.\n")
cmd.Flags().StringVar(&stepConfig.PathsIgnore, "pathsIgnore", os.Getenv("PIPER_pathsIgnore"), "List of file or directory patterns to ignore.\nEach entry must be on its own line, e.g.:\n **/*.md\n docs/**\nNote: This parameter is only applicable for interpreted languages.\n")
cmd.MarkFlagRequired("buildTool")
}
@@ -580,6 +584,24 @@ func codeqlExecuteScanMetadata() config.StepData {
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_transformQuerySuite"),
},
{
Name: "paths",
ResourceRef: []config.ResourceReference{},
Scope: []string{"STEPS", "STAGES", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_paths"),
},
{
Name: "pathsIgnore",
ResourceRef: []config.ResourceReference{},
Scope: []string{"STEPS", "STAGES", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_pathsIgnore"),
},
},
},
Containers: []config.Container{

View File

@@ -7,11 +7,15 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/SAP/jenkins-library/pkg/codeql"
"github.com/SAP/jenkins-library/pkg/mock"
@@ -1077,3 +1081,148 @@ func TestRunCustomCommand(t *testing.T) {
assert.Error(t, err)
})
}
func Test_prepareCodeQLConfigFile(t *testing.T) {
t.Run("creates_or_updates_default_config_next_to_codeql_binary", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("requires POSIX exec bits for the fake binary")
}
dir := t.TempDir()
codeqlBin := filepath.Join(dir, "codeql")
writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n")
origPath := os.Getenv("PATH")
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
opts := &codeqlExecuteScanOptions{
Paths: " src \nlib/utils\n",
PathsIgnore: "vendor\n**/*.gen.go\n",
}
err := prepareCodeQLConfigFile(opts)
require.NoError(t, err)
qlPath, err := codeql.Which("codeql")
require.NoError(t, err)
loc, found := strings.CutSuffix(qlPath, "codeql")
require.True(t, found)
cfgPath := path.Join(loc, "default-codeql-config.yml")
b, err := os.ReadFile(cfgPath)
require.NoError(t, err)
out := normalizeNL(string(b))
// Assert it contains the expected lists (don’t depend on ordering)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- src")
assert.Contains(t, out, "- lib/utils")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- vendor")
// yaml.v3 single-quotes strings with '*' etc.
assert.Contains(t, out, "- '**/*.gen.go'")
})
t.Run("when_codeql_binary_missing_returns_error", func(t *testing.T) {
dir := t.TempDir()
origPath := os.Getenv("PATH")
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
// PATH without codeql
require.NoError(t, os.Setenv("PATH", dir))
opts := &codeqlExecuteScanOptions{
Paths: "a",
PathsIgnore: "b",
}
err := prepareCodeQLConfigFile(opts)
require.Error(t, err)
assert.Contains(t, err.Error(), "could not locate codeql executable")
})
t.Run("propagates_append_error_when_default_config_path_is_a_directory", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("requires POSIX exec bits and reliable directory perms")
}
dir := t.TempDir()
// Create fake codeql binary
codeqlBin := filepath.Join(dir, "codeql")
writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n")
defaultCfgDir := filepath.Join(dir, "default-codeql-config.yml")
require.NoError(t, os.Mkdir(defaultCfgDir, 0o755))
origPath := os.Getenv("PATH")
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
opts := &codeqlExecuteScanOptions{
Paths: "x",
PathsIgnore: "y",
}
err := prepareCodeQLConfigFile(opts)
require.Error(t, err)
assert.Contains(t, err.Error(), "append paths and paths ignore to the default config")
})
t.Run("no_changes_when_both_paths_empty_but_still_needs_binary", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("requires POSIX exec bits for the fake binary")
}
dir := t.TempDir()
codeqlBin := filepath.Join(dir, "codeql")
writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n")
origPath := os.Getenv("PATH")
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
cfgFile := filepath.Join(dir, "default-codeql-config.yml")
writeCodeQLFile(t, cfgFile, "")
opts := &codeqlExecuteScanOptions{
Paths: "",
PathsIgnore: "",
}
before := readCodeQLFile(t, cfgFile)
time.Sleep(10 * time.Millisecond) // avoid flakiness on very fast FS
err := prepareCodeQLConfigFile(opts)
require.NoError(t, err)
after := readCodeQLFile(t, cfgFile)
assert.Equal(t, before, after, "file should remain unchanged when nothing to write")
})
}
func writeExec(t *testing.T, path, content string) {
t.Helper()
require.NoError(t, os.WriteFile(path, []byte(content), 0o755))
require.NoError(t, os.Chmod(path, 0o755))
info, err := os.Stat(path)
require.NoError(t, err)
require.False(t, info.IsDir())
require.NotZero(t, info.Mode()&0o111, "should be executable")
}
func writeCodeQLFile(t *testing.T, path, content string) {
t.Helper()
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
}
func readCodeQLFile(t *testing.T, path string) string {
t.Helper()
b, err := os.ReadFile(path)
require.NoError(t, err)
return string(b)
}
func normalizeNL(s string) string {
return strings.ReplaceAll(s, "\r\n", "\n")
}

View File

@@ -304,8 +304,8 @@ type System interface {
GetScanSummary(scanID string) (ScanSummary, error)
GetResultsPredicates(SimilarityID int64, ProjectID string) ([]ResultsPredicates, error)
GetScanWorkflow(scanID string) ([]WorkflowLog, error)
GetLastScans(projectID string, limit int) ([]Scan, error)
GetLastScansByStatus(projectID string, limit int, status []string) ([]Scan, error)
GetLastScans(projectID, branch string, limit int) ([]Scan, error)
GetLastScansByStatus(projectID, branch string, limit int, status []string) ([]Scan, error)
ScanProject(projectID, sourceUrl, branch, scanType string, settings []ScanConfiguration, tags map[string]string) (Scan, error)
ScanProjectZip(projectID, sourceUrl, branch string, settings []ScanConfiguration, tags map[string]string) (Scan, error)
@@ -1142,7 +1142,7 @@ func (sys *SystemInstance) GetScanWorkflow(scanID string) ([]WorkflowLog, error)
return workflow, nil
}
func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, error) {
func (sys *SystemInstance) GetLastScans(projectID, branch string, limit int) ([]Scan, error) {
var scanResponse struct {
TotalCount uint64
FilteredTotalCount uint64
@@ -1163,6 +1163,7 @@ func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, er
"offset": {fmt.Sprintf("%v", 0)},
"limit": {fmt.Sprintf("%v", limit)},
"sort": {sortStr},
"branches": []string{branch},
}
header := http.Header{}
@@ -1177,7 +1178,7 @@ func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, er
return scanResponse.Scans, err
}
func (sys *SystemInstance) GetLastScansByStatus(projectID string, limit int, status []string) ([]Scan, error) {
func (sys *SystemInstance) GetLastScansByStatus(projectID, branch string, limit int, status []string) ([]Scan, error) {
var scanResponse struct {
TotalCount uint64
FilteredTotalCount uint64
@@ -1199,6 +1200,7 @@ func (sys *SystemInstance) GetLastScansByStatus(projectID string, limit int, sta
"limit": {fmt.Sprintf("%d", limit)},
"sort": {sortStr},
"statuses": status,
"branches": []string{branch},
}
data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/scans/?%v", body.Encode()), nil, nil, []int{})

105
pkg/codeql/path.go Normal file
View File

@@ -0,0 +1,105 @@
package codeql
import (
"os"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
)
// AppendCodeQLPaths updates the CodeQL config YAML with new paths/paths-ignore.
func AppendCodeQLPaths(cfgPath string, scanPaths, ignorePaths []string) error {
if len(scanPaths) == 0 && len(ignorePaths) == 0 {
// if both paths are empty - do not touch anything
return nil
}
var cfg map[string]any
data, err := os.ReadFile(cfgPath)
if err != nil {
if !os.IsNotExist(err) {
return err
}
} else {
// file exists; unmarshal if non-empty
if len(data) > 0 {
if err = yaml.Unmarshal(data, &cfg); err != nil {
return err
}
}
}
if cfg == nil {
// start from empty config if file doesn't exist or empty
cfg = make(map[string]any)
}
if len(scanPaths) != 0 {
cfg["paths"] = scanPaths
}
if len(ignorePaths) != 0 {
cfg["paths-ignore"] = ignorePaths
}
out, err := yaml.Marshal(cfg)
if err != nil {
return err
}
return os.WriteFile(cfgPath, out, 0o644)
}
func ParsePaths(pathsStr string) []string {
var paths []string
patterns := strings.Split(pathsStr, "\n")
for _, p := range patterns {
p = strings.TrimSpace(p)
if p == "" {
continue
}
paths = append(paths, p)
}
return paths
}
// Which finds the first executable in PATH and resolves symlinks.
func Which(name string) (string, error) {
// If the name already has a path separator, check it directly.
if strings.ContainsRune(name, filepath.Separator) {
if isExecutable(name) {
return resolve(name)
}
return "", os.ErrNotExist
}
// Search in PATH
for _, dir := range filepath.SplitList(os.Getenv("PATH")) {
if dir == "" {
dir = "."
}
candidate := filepath.Join(dir, name)
if isExecutable(candidate) {
return resolve(candidate)
}
}
return "", os.ErrNotExist
}
func isExecutable(path string) bool {
info, err := os.Stat(path)
if err != nil {
return false
}
if info.IsDir() {
return false
}
// POSIX: check execute bit
return info.Mode()&0o111 != 0
}
func resolve(path string) (string, error) {
abs, err := filepath.Abs(path)
if err != nil {
return "", err
}
return filepath.EvalSymlinks(abs)
}

326
pkg/codeql/path_test.go Normal file
View File

@@ -0,0 +1,326 @@
package codeql
import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAppendCodeQLPaths(t *testing.T) {
t.Run("AppendCodeQLPaths", func(t *testing.T) {
t.Run("creates_file_if_not_exists_with_both_keys", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
err := AppendCodeQLPaths(cfgPath,
[]string{"src", "lib/utils"},
[]string{"vendor", "**/*.gen.go"},
)
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- src")
assert.Contains(t, out, "- lib/utils")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- vendor")
assert.Contains(t, out, "- '**/*.gen.go'") // YAML encoder uses single quotes for '*'
})
t.Run("creates_file_if_not_exists_with_only_paths", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
err := AppendCodeQLPaths(cfgPath, []string{"a", "b"}, nil)
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- a")
assert.Contains(t, out, "- b")
assert.NotContains(t, out, "paths-ignore:")
})
t.Run("creates_file_if_not_exists_with_only_paths_ignore", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
err := AppendCodeQLPaths(cfgPath, nil, []string{"vendor"})
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.NotContains(t, out, "paths:\n")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- vendor")
})
t.Run("appends_to_empty_yaml_creates_paths_and_paths_ignore", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
writeFile(t, cfgPath, "")
err := AppendCodeQLPaths(cfgPath,
[]string{"src", "lib/utils"},
[]string{"vendor", "**/*.gen.go"},
)
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- src")
assert.Contains(t, out, "- lib/utils")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- vendor")
assert.Contains(t, out, "- '**/*.gen.go'")
})
t.Run("overwrites_on_type_mismatch_and_preserves_other_keys", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
// Intentionally set scalar for paths to force overwrite; and keep an existing ignore.
initial := "paths: foo\npaths-ignore:\n - old-ignore\n"
writeFile(t, cfgPath, initial)
err := AppendCodeQLPaths(cfgPath, []string{"new-path"}, nil)
require.NoError(t, err)
out := readFile(t, cfgPath)
// 'paths' should now be a sequence containing only the new value
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- new-path")
assert.NotContains(t, out, "foo")
// paths-ignore should remain present (we didn't provide a new ignore list)
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- old-ignore")
})
t.Run("updates_only_paths_when_ignore_not_provided", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
initial := "paths:\n - old\npaths-ignore:\n - keep-ignore\n"
writeFile(t, cfgPath, initial)
err := AppendCodeQLPaths(cfgPath, []string{"new-a", "new-b"}, nil)
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- new-a")
assert.Contains(t, out, "- new-b")
assert.NotContains(t, out, "- old")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- keep-ignore")
})
t.Run("updates_only_paths_ignore_when_paths_not_provided", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
initial := "paths:\n - keep\npaths-ignore:\n - old-ignore\n"
writeFile(t, cfgPath, initial)
err := AppendCodeQLPaths(cfgPath, nil, []string{"new-ignore-1", "**/*.gen.go"})
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- keep")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- new-ignore-1")
assert.Contains(t, out, "- '**/*.gen.go'")
assert.NotContains(t, out, "- old-ignore")
})
t.Run("no_op_when_no_values_on_existing_file", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
initial := "paths:\n - keep\npaths-ignore:\n - keep-ignore\n"
writeFile(t, cfgPath, initial)
before := readFile(t, cfgPath)
// Touch mtime differences robustly across filesystems.
time.Sleep(10 * time.Millisecond)
err := AppendCodeQLPaths(cfgPath, nil, nil)
require.NoError(t, err)
after := readFile(t, cfgPath)
assert.Equal(t, before, after, "file content should be unchanged for no-op with nil slices")
})
t.Run("no_op_when_no_values_and_file_missing_does_not_create_file", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
err := AppendCodeQLPaths(cfgPath, []string{}, []string{})
require.NoError(t, err)
_, statErr := os.Stat(cfgPath)
require.True(t, os.IsNotExist(statErr), "file should not be created when both inputs are empty")
})
t.Run("works_with_existing_empty_file", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
writeFile(t, cfgPath, "")
err := AppendCodeQLPaths(cfgPath, []string{"p1"}, []string{"i1"})
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- p1")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- i1")
})
})
}
func TestParsePaths(t *testing.T) {
t.Run("ParsePaths", func(t *testing.T) {
cases := []struct {
name string
in string
out []string
}{
{
name: "trims_and_skips_blanks",
in: " src \n\nlib\n \nvendor ",
out: []string{"src", "lib", "vendor"},
},
{
name: "handles_single_line",
in: "only/one",
out: []string{"only/one"},
},
{
name: "keeps_globs_and_spaces_inside_line",
in: "**/*.go\npath with space/\n./rel",
out: []string{"**/*.go", "path with space/", "./rel"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := ParsePaths(tc.in)
assert.Equal(t, tc.out, got)
})
}
})
}
func TestWhich(t *testing.T) {
t.Run("Which", func(t *testing.T) {
// Save and restore PATH for isolation.
origPath := os.Getenv("PATH")
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
t.Run("nonexistent_returns_ErrNotExist", func(t *testing.T) {
_, err := Which("definitely-not-a-real-binary-name")
assert.ErrorIs(t, err, os.ErrNotExist)
})
t.Run("finds_executable_in_PATH", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("executable bit semantics differ on Windows; test skipped")
}
dir := t.TempDir()
bin := filepath.Join(dir, "mytool")
writeExec(t, bin, "#!/bin/sh\necho ok\n")
// Put our temp dir first on PATH
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
found, err := Which("mytool")
require.NoError(t, err)
// Compare against the resolved path (handles /var -> /private/var on macOS)
expected, err := resolve(bin)
require.NoError(t, err)
assert.Equal(t, expected, found)
})
t.Run("resolves_symlinks", func(t *testing.T) {
// Symlink behavior varies on Windows (needs admin/developer mode).
if runtime.GOOS == "windows" {
t.Skip("symlink creation is restricted on Windows; test skipped")
}
dir := t.TempDir()
target := filepath.Join(dir, "realtool")
link := filepath.Join(dir, "alias")
writeExec(t, target, "#!/bin/sh\necho real\n")
err := os.Symlink(target, link)
if err != nil {
t.Skipf("symlinks not supported in this environment: %v", err)
}
require.NoError(t, os.Setenv("PATH", dir))
found, err := Which("alias")
require.NoError(t, err)
expected, err := resolve(target)
require.NoError(t, err)
assert.Equal(t, expected, found, "Which should resolve through the symlink to the real path")
})
t.Run("direct_path_with_separator", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("executable bit semantics differ on Windows; test skipped")
}
dir := t.TempDir()
bin := filepath.Join(dir, "runme")
writeExec(t, bin, "#!/bin/sh\necho hi\n")
found, err := Which(bin)
require.NoError(t, err)
expected, err := resolve(bin)
require.NoError(t, err)
assert.Equal(t, expected, found)
})
})
}
func writeFile(t *testing.T, path, content string) {
t.Helper()
err := os.WriteFile(path, []byte(content), 0o644)
require.NoError(t, err)
}
func writeExec(t *testing.T, path, content string) {
t.Helper()
err := os.WriteFile(path, []byte(content), 0o755)
require.NoError(t, err)
// On some systems, writing then chmod is safer/more explicit.
err = os.Chmod(path, 0o755)
require.NoError(t, err)
// Sanity: ensure it's not a directory and has any execute bit set (for non-Windows cases we test)
info, err := os.Stat(path)
require.NoError(t, err)
require.False(t, info.IsDir())
require.True(t, info.Mode()&0o111 != 0)
}
func readFile(t *testing.T, path string) string {
t.Helper()
b, err := os.ReadFile(path)
require.NoError(t, err)
// Normalize Windows newlines just in case, so substring checks are reliable.
return strings.ReplaceAll(string(b), "\r\n", "\n")
}

View File

@@ -247,6 +247,31 @@ spec:
- STEPS
- STAGES
- PARAMETERS
- name: paths
type: string
description: |
List of file or directory patterns to include.
Each entry must be on its own line, e.g.:
src/**
lib/**
Note: This parameter is only applicable for interpreted languages.
scope:
- STEPS
- STAGES
- PARAMETERS
- name: pathsIgnore
type: string
pathsIgnore:
description: |
List of file or directory patterns to ignore.
Each entry must be on its own line, e.g.:
**/*.md
docs/**
Note: This parameter is only applicable for interpreted languages.
scope:
- STEPS
- STAGES
- PARAMETERS
containers:
- image: ""
outputs: