From 3cc0e99d11d7c69cfa0e250e7e4e40de94380070 Mon Sep 17 00:00:00 2001 From: Adrien <99400874+hubadr@users.noreply.github.com> Date: Thu, 11 Sep 2025 09:55:04 +0200 Subject: [PATCH 1/3] Feat(CxOne): incremental scans based on another branch (#5476) --- cmd/checkmarxOneExecuteScan.go | 92 ++++++++++++++++++++++------- cmd/checkmarxOneExecuteScan_test.go | 4 +- pkg/checkmarxone/checkmarxone.go | 10 ++-- 3 files changed, 78 insertions(+), 28 deletions(-) diff --git a/cmd/checkmarxOneExecuteScan.go b/cmd/checkmarxOneExecuteScan.go index 73fa37fd3..67a637135 100644 --- a/cmd/checkmarxOneExecuteScan.go +++ b/cmd/checkmarxOneExecuteScan.go @@ -128,7 +128,8 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS } } - scans, err := cx1sh.GetLastScans(10) + branch, isPR, baseBranch := cx1sh.GetScanBranch() + scans, err := cx1sh.GetLastScans(10, branch) if err != nil { log.Entry().WithError(err).Warnf("failed to get last 10 scans") } @@ -158,7 +159,7 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS } if config.Incremental { - log.Entry().Warnf("If you change your file filter pattern it is recommended to run a Full scan instead of an incremental, to ensure full code coverage.") + log.Entry().Info("If you change your file filter pattern it is recommended to run a Full scan instead of an incremental, to ensure full code coverage.") } zipFile, err := cx1sh.ZipFiles() @@ -172,7 +173,40 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS } // TODO : The step structure should allow to enable different scanners: SAST, KICKS, SCA - scan, err := cx1sh.CreateScanRequest(incremental, uploadLink) + var scan *checkmarxOne.Scan + // We search for a full scan in history: + fullScanExists := false + for _, histScan := range scans { + isIncr, _ := histScan.IsIncremental() + if isIncr == false { + fullScanExists = true + break + } + } + // user requested an incremental scan on a branch, and the project has a Primary Branch set, not in PR context and no full scan on the branch + if config.Incremental && !isPR && !fullScanExists && cx1sh.Project.MainBranch != "" && cx1sh.Project.MainBranch != branch { + scansMainBranch, err := cx1sh.GetLastScans(10, cx1sh.Project.MainBranch) + if err != nil { + return fmt.Errorf("failed to determine incremental or full scan configuration: %s", err) + } + // We check if the main branch is eligible for an incremental scan + incrementalMainBranch, err := cx1sh.IncrementalOrFull(scansMainBranch) + log.Entry().Debugf("Main branch %v incremental scan eligibility: %t", cx1sh.Project.MainBranch, incrementalMainBranch) + scan, err = cx1sh.CreateScanRequest(incrementalMainBranch, uploadLink, cx1sh.Project.MainBranch) // this will create a full scan on the current branch if the main branch is not eligible for an incremental scan + } else if config.Incremental && isPR && len(baseBranch) > 0 && baseBranch != "n/a" { // running in a PR context, and we have a base branch for the incremental scan + // in a PR context we always want to do an incremental scan + // The scan will be based on the PR's target branch (baseBranch) if there is no full scan on the PR branch + if fullScanExists { + log.Entry().Debugf("A full scan exists on the PR branch %v, so the incremental scan will be based on it", branch) + scan, err = cx1sh.CreateScanRequest(true, uploadLink, "") + } else { + log.Entry().Debugf("There is no full scan on the PR branch %v, so the incremental scan will be based on branch %v", branch, baseBranch) + scan, err = cx1sh.CreateScanRequest(true, uploadLink, baseBranch) + } + } else { + scan, err = cx1sh.CreateScanRequest(incremental, uploadLink, "") + } + if err != nil { return fmt.Errorf("failed to create scan: %s", err) } @@ -388,8 +422,8 @@ func (c *checkmarxOneExecuteScanHelper) SetProjectPreset() error { return nil } -func (c *checkmarxOneExecuteScanHelper) GetLastScans(count int) ([]checkmarxOne.Scan, error) { - scans, err := c.sys.GetLastScansByStatus(c.Project.ProjectID, count, []string{"Completed"}) +func (c *checkmarxOneExecuteScanHelper) GetLastScans(count int, branch string) ([]checkmarxOne.Scan, error) { + scans, err := c.sys.GetLastScansByStatus(c.Project.ProjectID, branch, count, []string{"Completed"}) if err != nil { return []checkmarxOne.Scan{}, fmt.Errorf("Failed to get last %d Completed scans for project %v: %s", count, c.Project.ProjectID, err) } @@ -437,23 +471,10 @@ func (c *checkmarxOneExecuteScanHelper) UploadScanContent(zipFile *os.File) (str return uploadUri, nil } -func (c *checkmarxOneExecuteScanHelper) CreateScanRequest(incremental bool, uploadLink string) (*checkmarxOne.Scan, error) { - sastConfig := checkmarxOne.ScanConfiguration{} - sastConfig.ScanType = "sast" - - sastConfig.Values = make(map[string]string, 0) - sastConfig.Values["incremental"] = strconv.FormatBool(incremental) - sastConfig.Values["presetName"] = c.config.Preset // always set, either coming from config or coming from Cx1 configuration - sastConfigString := fmt.Sprintf("incremental %v, preset %v", strconv.FormatBool(incremental), c.config.Preset) - - if len(c.config.LanguageMode) > 0 { - sastConfig.Values["languageMode"] = c.config.LanguageMode - sastConfigString = sastConfigString + fmt.Sprintf(", languageMode %v", c.config.LanguageMode) - } - +func (c *checkmarxOneExecuteScanHelper) GetScanBranch() (string, bool, string) { branch := c.config.Branch cicdOrch, err := orchestrator.GetOrchestratorConfigProvider(nil) - if err == nil { + if err != nil { log.Entry().Warn("Could not identify orchestrator") } if len(branch) == 0 && len(c.config.GitBranch) > 0 && c.config.GitBranch != "n/a" { @@ -462,17 +483,43 @@ func (c *checkmarxOneExecuteScanHelper) CreateScanRequest(incremental bool, uplo cicdBranch := cicdOrch.Branch() if cicdBranch != "n/a" { branch = cicdBranch - log.Entry().Infof("CxOne scan branch was automatically set to : %v", branch) } else { log.Entry().Info("Could not retrieve branch name from orchestrator") } } if len(c.config.PullRequestName) > 0 { - branch = fmt.Sprintf("%v-%v", c.config.PullRequestName, c.config.Branch) + branch = fmt.Sprintf("%v-%v", c.config.PullRequestName, branch) } else if cicdOrch.IsPullRequest() && cicdOrch.PullRequestConfig().Branch != "n/a" { branch = fmt.Sprintf("PR%v-%v", cicdOrch.PullRequestConfig().Key, cicdOrch.PullRequestConfig().Branch) } + baseBranch := cicdOrch.PullRequestConfig().Base + isPR := cicdOrch.IsPullRequest() + log.Entry().Debugf("CxOne scan branch was automatically set to : %v", branch) + return branch, isPR, baseBranch +} + +func (c *checkmarxOneExecuteScanHelper) CreateScanRequest(incremental bool, uploadLink string, baseBranch string) (*checkmarxOne.Scan, error) { + sastConfigString := "" + sastConfig := checkmarxOne.ScanConfiguration{} + sastConfig.ScanType = "sast" + + sastConfig.Values = make(map[string]string, 0) + sastConfig.Values["incremental"] = strconv.FormatBool(incremental) + sastConfig.Values["presetName"] = c.config.Preset // always set, either coming from config or coming from Cx1 configuration + if incremental && len(baseBranch) > 0 { // base the incremental scan on the specified base branch + sastConfig.Values["baseBranch"] = baseBranch + sastConfigString = fmt.Sprintf("baseBranch: %v, ", baseBranch) + } + sastConfigString = fmt.Sprintf("%vincremental %v, preset %v", sastConfigString, strconv.FormatBool(incremental), c.config.Preset) + + if len(c.config.LanguageMode) > 0 { + sastConfig.Values["languageMode"] = c.config.LanguageMode + sastConfigString = sastConfigString + fmt.Sprintf(", languageMode %v", c.config.LanguageMode) + } + + branch, _, _ := c.GetScanBranch() + sastConfigString = fmt.Sprintf("Cx1 Branch name %v, ", branch) + sastConfigString log.Entry().Infof("Will run a scan with the following configuration: %v", sastConfigString) @@ -1028,6 +1075,7 @@ func (c *checkmarxOneExecuteScanHelper) getDetailedResults(scan *checkmarxOne.Sc func (c *checkmarxOneExecuteScanHelper) zipWorkspaceFiles(filterPattern string, utils checkmarxOneExecuteScanUtils) (*os.File, error) { zipFileName := filepath.Join(utils.GetWorkspace(), "workspace.zip") + log.Entry().Infof("Zipping files using filter: %v", filterPattern) patterns := piperutils.Trim(strings.Split(filterPattern, ",")) sort.Strings(patterns) zipFile, err := os.Create(zipFileName) diff --git a/cmd/checkmarxOneExecuteScan_test.go b/cmd/checkmarxOneExecuteScan_test.go index c9206dbf8..4bf20c3d3 100644 --- a/cmd/checkmarxOneExecuteScan_test.go +++ b/cmd/checkmarxOneExecuteScan_test.go @@ -70,11 +70,11 @@ func (sys *checkmarxOneSystemMock) GetScanWorkflow(scanID string) ([]checkmarxOn return []checkmarxOne.WorkflowLog{}, nil } -func (sys *checkmarxOneSystemMock) GetLastScans(projectID string, limit int) ([]checkmarxOne.Scan, error) { +func (sys *checkmarxOneSystemMock) GetLastScans(projectID, branch string, limit int) ([]checkmarxOne.Scan, error) { return []checkmarxOne.Scan{}, nil } -func (sys *checkmarxOneSystemMock) GetLastScansByStatus(projectID string, limit int, status []string) ([]checkmarxOne.Scan, error) { +func (sys *checkmarxOneSystemMock) GetLastScansByStatus(projectID, branch string, limit int, status []string) ([]checkmarxOne.Scan, error) { return []checkmarxOne.Scan{}, nil } diff --git a/pkg/checkmarxone/checkmarxone.go b/pkg/checkmarxone/checkmarxone.go index f5be1358f..940a79359 100644 --- a/pkg/checkmarxone/checkmarxone.go +++ b/pkg/checkmarxone/checkmarxone.go @@ -304,8 +304,8 @@ type System interface { GetScanSummary(scanID string) (ScanSummary, error) GetResultsPredicates(SimilarityID int64, ProjectID string) ([]ResultsPredicates, error) GetScanWorkflow(scanID string) ([]WorkflowLog, error) - GetLastScans(projectID string, limit int) ([]Scan, error) - GetLastScansByStatus(projectID string, limit int, status []string) ([]Scan, error) + GetLastScans(projectID, branch string, limit int) ([]Scan, error) + GetLastScansByStatus(projectID, branch string, limit int, status []string) ([]Scan, error) ScanProject(projectID, sourceUrl, branch, scanType string, settings []ScanConfiguration, tags map[string]string) (Scan, error) ScanProjectZip(projectID, sourceUrl, branch string, settings []ScanConfiguration, tags map[string]string) (Scan, error) @@ -1142,7 +1142,7 @@ func (sys *SystemInstance) GetScanWorkflow(scanID string) ([]WorkflowLog, error) return workflow, nil } -func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, error) { +func (sys *SystemInstance) GetLastScans(projectID, branch string, limit int) ([]Scan, error) { var scanResponse struct { TotalCount uint64 FilteredTotalCount uint64 @@ -1163,6 +1163,7 @@ func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, er "offset": {fmt.Sprintf("%v", 0)}, "limit": {fmt.Sprintf("%v", limit)}, "sort": {sortStr}, + "branches": []string{branch}, } header := http.Header{} @@ -1177,7 +1178,7 @@ func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, er return scanResponse.Scans, err } -func (sys *SystemInstance) GetLastScansByStatus(projectID string, limit int, status []string) ([]Scan, error) { +func (sys *SystemInstance) GetLastScansByStatus(projectID, branch string, limit int, status []string) ([]Scan, error) { var scanResponse struct { TotalCount uint64 FilteredTotalCount uint64 @@ -1199,6 +1200,7 @@ func (sys *SystemInstance) GetLastScansByStatus(projectID string, limit int, sta "limit": {fmt.Sprintf("%d", limit)}, "sort": {sortStr}, "statuses": status, + "branches": []string{branch}, } data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/scans/?%v", body.Encode()), nil, nil, []int{}) From b042ba490e0da9c613a8328690616ad6a75137d4 Mon Sep 17 00:00:00 2001 From: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Date: Fri, 12 Sep 2025 09:37:24 +0200 Subject: [PATCH 2/3] chore(renovate): add rule to update python Docker image (#5474) --- .github/renovate.json | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/renovate.json b/.github/renovate.json index 143269024..e8666e140 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -104,6 +104,34 @@ "depNameTemplate": "CycloneDX/{{{depName}}}", "extractVersionTemplate": "v(?.*)", "autoReplaceStringTemplate": "\"github.com/{{{depName}}}/cmd/cyclonedx-gomod@v{{{newValue}}}\"" + }, + { + "customType": "regex", + "matchStringsStrategy": "any", + "managerFilePatterns": [ + "resources/metadata/pythonBuild.yaml" + ], + "matchStrings": [ + "image: (?python):(?.*)" + ], + "depTypeTemplate": "dependencies", + "datasourceTemplate": "docker", + "extractVersionTemplate": "(?.*)", + "autoReplaceStringTemplate": "image: {{depName}}:{{newValue}}" + }, + { + "customType": "regex", + "matchStringsStrategy": "any", + "managerFilePatterns": [ + "cmd/pythonBuild_generated.go" + ], + "matchStrings": [ + "Image: \"(?python):(?.*)\"" + ], + "depTypeTemplate": "dependencies", + "datasourceTemplate": "docker", + "extractVersionTemplate": "(?.*)", + "autoReplaceStringTemplate": "Image: \"{{depName}}:{{newValue}}\"" } ], "postUpdateOptions": [ From 48582710bd47ba0c03aa8bc270e5d82433dbd4a0 Mon Sep 17 00:00:00 2001 From: Timur Akhmadiev <42980811+Sugar-pack@users.noreply.github.com> Date: Fri, 12 Sep 2025 19:37:30 +0400 Subject: [PATCH 3/3] feat(codeql): ignore paths and scan paths (#5477) --- cmd/codeqlExecuteScan.go | 28 ++ cmd/codeqlExecuteScan_generated.go | 22 ++ cmd/codeqlExecuteScan_test.go | 149 ++++++++++ pkg/codeql/path.go | 105 +++++++ pkg/codeql/path_test.go | 326 ++++++++++++++++++++++ resources/metadata/codeqlExecuteScan.yaml | 25 ++ 6 files changed, 655 insertions(+) create mode 100644 pkg/codeql/path.go create mode 100644 pkg/codeql/path_test.go diff --git a/cmd/codeqlExecuteScan.go b/cmd/codeqlExecuteScan.go index 18ce5bc8d..b23943f37 100644 --- a/cmd/codeqlExecuteScan.go +++ b/cmd/codeqlExecuteScan.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "os" + "path" "path/filepath" "strings" @@ -136,11 +137,38 @@ func printCodeqlImageVersion() { } } +func prepareCodeQLConfigFile(config *codeqlExecuteScanOptions) error { + pathsToScan := codeql.ParsePaths(config.Paths) + pathsToIgnore := codeql.ParsePaths(config.PathsIgnore) + + codeQLExecName := "codeql" + codeQLPath, err := codeql.Which(codeQLExecName) + if err != nil { + return fmt.Errorf("could not locate codeql executable %w", err) + } + location, fileName := path.Split(codeQLPath) + if fileName != codeQLExecName { + return fmt.Errorf("could not find codeql executable in path: %s", codeQLPath) + } + defaultConfigLocation := path.Join(location, "default-codeql-config.yml") + err = codeql.AppendCodeQLPaths(defaultConfigLocation, pathsToScan, pathsToIgnore) + if err != nil { + return fmt.Errorf("append paths and paths ignore to the default config: %w", err) + } + return nil +} + func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, utils codeqlExecuteScanUtils, influx *codeqlExecuteScanInflux) ([]piperutils.Path, error) { printCodeqlImageVersion() var reports []piperutils.Path + err := prepareCodeQLConfigFile(config) + if err != nil { + log.Entry().WithError(err).Error("failed to prepare codeql config file") + return reports, err + } + dbCreateCustomFlags := codeql.ParseCustomFlags(config.DatabaseCreateFlags) isMultiLang, err := runDatabaseCreate(config, dbCreateCustomFlags, utils) if err != nil { diff --git a/cmd/codeqlExecuteScan_generated.go b/cmd/codeqlExecuteScan_generated.go index 183381f8f..2afce9459 100644 --- a/cmd/codeqlExecuteScan_generated.go +++ b/cmd/codeqlExecuteScan_generated.go @@ -48,6 +48,8 @@ type codeqlExecuteScanOptions struct { DatabaseAnalyzeFlags string `json:"databaseAnalyzeFlags,omitempty"` CustomCommand string `json:"customCommand,omitempty"` TransformQuerySuite string `json:"transformQuerySuite,omitempty"` + Paths string `json:"paths,omitempty"` + PathsIgnore string `json:"pathsIgnore,omitempty"` } type codeqlExecuteScanInflux struct { @@ -306,6 +308,8 @@ func addCodeqlExecuteScanFlags(cmd *cobra.Command, stepConfig *codeqlExecuteScan cmd.Flags().StringVar(&stepConfig.DatabaseAnalyzeFlags, "databaseAnalyzeFlags", os.Getenv("PIPER_databaseAnalyzeFlags"), "A space-separated string of flags for the 'codeql database analyze' command.") cmd.Flags().StringVar(&stepConfig.CustomCommand, "customCommand", os.Getenv("PIPER_customCommand"), "A custom user-defined command to run between codeql analysis and results upload.") cmd.Flags().StringVar(&stepConfig.TransformQuerySuite, "transformQuerySuite", os.Getenv("PIPER_transformQuerySuite"), "A transform string that will be applied to the querySuite using the sed command.") + cmd.Flags().StringVar(&stepConfig.Paths, "paths", os.Getenv("PIPER_paths"), "List of file or directory patterns to include.\nEach entry must be on its own line, e.g.:\n src/**\n lib/**\nNote: This parameter is only applicable for interpreted languages.\n") + cmd.Flags().StringVar(&stepConfig.PathsIgnore, "pathsIgnore", os.Getenv("PIPER_pathsIgnore"), "List of file or directory patterns to ignore.\nEach entry must be on its own line, e.g.:\n **/*.md\n docs/**\nNote: This parameter is only applicable for interpreted languages.\n") cmd.MarkFlagRequired("buildTool") } @@ -580,6 +584,24 @@ func codeqlExecuteScanMetadata() config.StepData { Aliases: []config.Alias{}, Default: os.Getenv("PIPER_transformQuerySuite"), }, + { + Name: "paths", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_paths"), + }, + { + Name: "pathsIgnore", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_pathsIgnore"), + }, }, }, Containers: []config.Container{ diff --git a/cmd/codeqlExecuteScan_test.go b/cmd/codeqlExecuteScan_test.go index 4bf29e3a1..2113c6a07 100644 --- a/cmd/codeqlExecuteScan_test.go +++ b/cmd/codeqlExecuteScan_test.go @@ -7,11 +7,15 @@ import ( "fmt" "io" "os" + "path" "path/filepath" + "runtime" "strings" "testing" + "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/SAP/jenkins-library/pkg/codeql" "github.com/SAP/jenkins-library/pkg/mock" @@ -1077,3 +1081,148 @@ func TestRunCustomCommand(t *testing.T) { assert.Error(t, err) }) } + +func Test_prepareCodeQLConfigFile(t *testing.T) { + t.Run("creates_or_updates_default_config_next_to_codeql_binary", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("requires POSIX exec bits for the fake binary") + } + dir := t.TempDir() + + codeqlBin := filepath.Join(dir, "codeql") + writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n") + + origPath := os.Getenv("PATH") + t.Cleanup(func() { _ = os.Setenv("PATH", origPath) }) + require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath)) + + opts := &codeqlExecuteScanOptions{ + Paths: " src \nlib/utils\n", + PathsIgnore: "vendor\n**/*.gen.go\n", + } + + err := prepareCodeQLConfigFile(opts) + require.NoError(t, err) + + qlPath, err := codeql.Which("codeql") + require.NoError(t, err) + + loc, found := strings.CutSuffix(qlPath, "codeql") + require.True(t, found) + + cfgPath := path.Join(loc, "default-codeql-config.yml") + b, err := os.ReadFile(cfgPath) + require.NoError(t, err) + out := normalizeNL(string(b)) + + // Assert it contains the expected lists (don’t depend on ordering) + assert.Contains(t, out, "paths:") + assert.Contains(t, out, "- src") + assert.Contains(t, out, "- lib/utils") + + assert.Contains(t, out, "paths-ignore:") + assert.Contains(t, out, "- vendor") + // yaml.v3 single-quotes strings with '*' etc. + assert.Contains(t, out, "- '**/*.gen.go'") + }) + + t.Run("when_codeql_binary_missing_returns_error", func(t *testing.T) { + dir := t.TempDir() + origPath := os.Getenv("PATH") + t.Cleanup(func() { _ = os.Setenv("PATH", origPath) }) + + // PATH without codeql + require.NoError(t, os.Setenv("PATH", dir)) + + opts := &codeqlExecuteScanOptions{ + Paths: "a", + PathsIgnore: "b", + } + err := prepareCodeQLConfigFile(opts) + require.Error(t, err) + assert.Contains(t, err.Error(), "could not locate codeql executable") + }) + + t.Run("propagates_append_error_when_default_config_path_is_a_directory", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("requires POSIX exec bits and reliable directory perms") + } + + dir := t.TempDir() + + // Create fake codeql binary + codeqlBin := filepath.Join(dir, "codeql") + writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n") + + defaultCfgDir := filepath.Join(dir, "default-codeql-config.yml") + require.NoError(t, os.Mkdir(defaultCfgDir, 0o755)) + + origPath := os.Getenv("PATH") + t.Cleanup(func() { _ = os.Setenv("PATH", origPath) }) + require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath)) + + opts := &codeqlExecuteScanOptions{ + Paths: "x", + PathsIgnore: "y", + } + err := prepareCodeQLConfigFile(opts) + require.Error(t, err) + assert.Contains(t, err.Error(), "append paths and paths ignore to the default config") + }) + + t.Run("no_changes_when_both_paths_empty_but_still_needs_binary", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("requires POSIX exec bits for the fake binary") + } + dir := t.TempDir() + + codeqlBin := filepath.Join(dir, "codeql") + writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n") + origPath := os.Getenv("PATH") + t.Cleanup(func() { _ = os.Setenv("PATH", origPath) }) + require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath)) + + cfgFile := filepath.Join(dir, "default-codeql-config.yml") + writeCodeQLFile(t, cfgFile, "") + + opts := &codeqlExecuteScanOptions{ + Paths: "", + PathsIgnore: "", + } + + before := readCodeQLFile(t, cfgFile) + time.Sleep(10 * time.Millisecond) // avoid flakiness on very fast FS + + err := prepareCodeQLConfigFile(opts) + require.NoError(t, err) + + after := readCodeQLFile(t, cfgFile) + assert.Equal(t, before, after, "file should remain unchanged when nothing to write") + }) +} + +func writeExec(t *testing.T, path, content string) { + t.Helper() + require.NoError(t, os.WriteFile(path, []byte(content), 0o755)) + require.NoError(t, os.Chmod(path, 0o755)) + info, err := os.Stat(path) + require.NoError(t, err) + require.False(t, info.IsDir()) + require.NotZero(t, info.Mode()&0o111, "should be executable") +} + +func writeCodeQLFile(t *testing.T, path, content string) { + t.Helper() + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) +} + +func readCodeQLFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + require.NoError(t, err) + return string(b) +} + +func normalizeNL(s string) string { + return strings.ReplaceAll(s, "\r\n", "\n") +} diff --git a/pkg/codeql/path.go b/pkg/codeql/path.go new file mode 100644 index 000000000..c98922637 --- /dev/null +++ b/pkg/codeql/path.go @@ -0,0 +1,105 @@ +package codeql + +import ( + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +// AppendCodeQLPaths updates the CodeQL config YAML with new paths/paths-ignore. +func AppendCodeQLPaths(cfgPath string, scanPaths, ignorePaths []string) error { + if len(scanPaths) == 0 && len(ignorePaths) == 0 { + // if both paths are empty - do not touch anything + return nil + } + var cfg map[string]any + + data, err := os.ReadFile(cfgPath) + if err != nil { + if !os.IsNotExist(err) { + return err + } + } else { + // file exists; unmarshal if non-empty + if len(data) > 0 { + if err = yaml.Unmarshal(data, &cfg); err != nil { + return err + } + } + } + if cfg == nil { + // start from empty config if file doesn't exist or empty + cfg = make(map[string]any) + } + + if len(scanPaths) != 0 { + cfg["paths"] = scanPaths + } + if len(ignorePaths) != 0 { + cfg["paths-ignore"] = ignorePaths + } + + out, err := yaml.Marshal(cfg) + if err != nil { + return err + } + return os.WriteFile(cfgPath, out, 0o644) +} + +func ParsePaths(pathsStr string) []string { + var paths []string + patterns := strings.Split(pathsStr, "\n") + for _, p := range patterns { + p = strings.TrimSpace(p) + if p == "" { + continue + } + paths = append(paths, p) + } + return paths +} + +// Which finds the first executable in PATH and resolves symlinks. +func Which(name string) (string, error) { + // If the name already has a path separator, check it directly. + if strings.ContainsRune(name, filepath.Separator) { + if isExecutable(name) { + return resolve(name) + } + return "", os.ErrNotExist + } + + // Search in PATH + for _, dir := range filepath.SplitList(os.Getenv("PATH")) { + if dir == "" { + dir = "." + } + candidate := filepath.Join(dir, name) + if isExecutable(candidate) { + return resolve(candidate) + } + } + return "", os.ErrNotExist +} + +func isExecutable(path string) bool { + info, err := os.Stat(path) + if err != nil { + return false + } + if info.IsDir() { + return false + } + // POSIX: check execute bit + return info.Mode()&0o111 != 0 +} + +func resolve(path string) (string, error) { + abs, err := filepath.Abs(path) + if err != nil { + return "", err + } + return filepath.EvalSymlinks(abs) +} diff --git a/pkg/codeql/path_test.go b/pkg/codeql/path_test.go new file mode 100644 index 000000000..a3c42f9d1 --- /dev/null +++ b/pkg/codeql/path_test.go @@ -0,0 +1,326 @@ +package codeql + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAppendCodeQLPaths(t *testing.T) { + t.Run("AppendCodeQLPaths", func(t *testing.T) { + t.Run("creates_file_if_not_exists_with_both_keys", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "codeql.yml") + + err := AppendCodeQLPaths(cfgPath, + []string{"src", "lib/utils"}, + []string{"vendor", "**/*.gen.go"}, + ) + require.NoError(t, err) + + out := readFile(t, cfgPath) + assert.Contains(t, out, "paths:") + assert.Contains(t, out, "- src") + assert.Contains(t, out, "- lib/utils") + + assert.Contains(t, out, "paths-ignore:") + assert.Contains(t, out, "- vendor") + assert.Contains(t, out, "- '**/*.gen.go'") // YAML encoder uses single quotes for '*' + }) + + t.Run("creates_file_if_not_exists_with_only_paths", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "codeql.yml") + + err := AppendCodeQLPaths(cfgPath, []string{"a", "b"}, nil) + require.NoError(t, err) + + out := readFile(t, cfgPath) + assert.Contains(t, out, "paths:") + assert.Contains(t, out, "- a") + assert.Contains(t, out, "- b") + assert.NotContains(t, out, "paths-ignore:") + }) + + t.Run("creates_file_if_not_exists_with_only_paths_ignore", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "codeql.yml") + + err := AppendCodeQLPaths(cfgPath, nil, []string{"vendor"}) + require.NoError(t, err) + + out := readFile(t, cfgPath) + assert.NotContains(t, out, "paths:\n") + assert.Contains(t, out, "paths-ignore:") + assert.Contains(t, out, "- vendor") + }) + + t.Run("appends_to_empty_yaml_creates_paths_and_paths_ignore", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "codeql.yml") + writeFile(t, cfgPath, "") + + err := AppendCodeQLPaths(cfgPath, + []string{"src", "lib/utils"}, + []string{"vendor", "**/*.gen.go"}, + ) + require.NoError(t, err) + + out := readFile(t, cfgPath) + + assert.Contains(t, out, "paths:") + assert.Contains(t, out, "- src") + assert.Contains(t, out, "- lib/utils") + + assert.Contains(t, out, "paths-ignore:") + assert.Contains(t, out, "- vendor") + assert.Contains(t, out, "- '**/*.gen.go'") + }) + + t.Run("overwrites_on_type_mismatch_and_preserves_other_keys", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "codeql.yml") + // Intentionally set scalar for paths to force overwrite; and keep an existing ignore. + initial := "paths: foo\npaths-ignore:\n - old-ignore\n" + writeFile(t, cfgPath, initial) + + err := AppendCodeQLPaths(cfgPath, []string{"new-path"}, nil) + require.NoError(t, err) + + out := readFile(t, cfgPath) + + // 'paths' should now be a sequence containing only the new value + assert.Contains(t, out, "paths:") + assert.Contains(t, out, "- new-path") + assert.NotContains(t, out, "foo") + + // paths-ignore should remain present (we didn't provide a new ignore list) + assert.Contains(t, out, "paths-ignore:") + assert.Contains(t, out, "- old-ignore") + }) + + t.Run("updates_only_paths_when_ignore_not_provided", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "codeql.yml") + initial := "paths:\n - old\npaths-ignore:\n - keep-ignore\n" + writeFile(t, cfgPath, initial) + + err := AppendCodeQLPaths(cfgPath, []string{"new-a", "new-b"}, nil) + require.NoError(t, err) + + out := readFile(t, cfgPath) + assert.Contains(t, out, "paths:") + assert.Contains(t, out, "- new-a") + assert.Contains(t, out, "- new-b") + assert.NotContains(t, out, "- old") + + assert.Contains(t, out, "paths-ignore:") + assert.Contains(t, out, "- keep-ignore") + }) + + t.Run("updates_only_paths_ignore_when_paths_not_provided", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "codeql.yml") + initial := "paths:\n - keep\npaths-ignore:\n - old-ignore\n" + writeFile(t, cfgPath, initial) + + err := AppendCodeQLPaths(cfgPath, nil, []string{"new-ignore-1", "**/*.gen.go"}) + require.NoError(t, err) + + out := readFile(t, cfgPath) + assert.Contains(t, out, "paths:") + assert.Contains(t, out, "- keep") + + assert.Contains(t, out, "paths-ignore:") + assert.Contains(t, out, "- new-ignore-1") + assert.Contains(t, out, "- '**/*.gen.go'") + assert.NotContains(t, out, "- old-ignore") + }) + + t.Run("no_op_when_no_values_on_existing_file", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "codeql.yml") + initial := "paths:\n - keep\npaths-ignore:\n - keep-ignore\n" + writeFile(t, cfgPath, initial) + before := readFile(t, cfgPath) + + // Touch mtime differences robustly across filesystems. + time.Sleep(10 * time.Millisecond) + + err := AppendCodeQLPaths(cfgPath, nil, nil) + require.NoError(t, err) + + after := readFile(t, cfgPath) + assert.Equal(t, before, after, "file content should be unchanged for no-op with nil slices") + }) + + t.Run("no_op_when_no_values_and_file_missing_does_not_create_file", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "codeql.yml") + + err := AppendCodeQLPaths(cfgPath, []string{}, []string{}) + require.NoError(t, err) + + _, statErr := os.Stat(cfgPath) + require.True(t, os.IsNotExist(statErr), "file should not be created when both inputs are empty") + }) + + t.Run("works_with_existing_empty_file", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "codeql.yml") + writeFile(t, cfgPath, "") + + err := AppendCodeQLPaths(cfgPath, []string{"p1"}, []string{"i1"}) + require.NoError(t, err) + + out := readFile(t, cfgPath) + assert.Contains(t, out, "paths:") + assert.Contains(t, out, "- p1") + assert.Contains(t, out, "paths-ignore:") + assert.Contains(t, out, "- i1") + }) + }) +} + +func TestParsePaths(t *testing.T) { + t.Run("ParsePaths", func(t *testing.T) { + cases := []struct { + name string + in string + out []string + }{ + { + name: "trims_and_skips_blanks", + in: " src \n\nlib\n \nvendor ", + out: []string{"src", "lib", "vendor"}, + }, + { + name: "handles_single_line", + in: "only/one", + out: []string{"only/one"}, + }, + { + name: "keeps_globs_and_spaces_inside_line", + in: "**/*.go\npath with space/\n./rel", + out: []string{"**/*.go", "path with space/", "./rel"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ParsePaths(tc.in) + assert.Equal(t, tc.out, got) + }) + } + }) +} + +func TestWhich(t *testing.T) { + t.Run("Which", func(t *testing.T) { + // Save and restore PATH for isolation. + origPath := os.Getenv("PATH") + t.Cleanup(func() { _ = os.Setenv("PATH", origPath) }) + + t.Run("nonexistent_returns_ErrNotExist", func(t *testing.T) { + _, err := Which("definitely-not-a-real-binary-name") + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("finds_executable_in_PATH", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("executable bit semantics differ on Windows; test skipped") + } + dir := t.TempDir() + bin := filepath.Join(dir, "mytool") + writeExec(t, bin, "#!/bin/sh\necho ok\n") + + // Put our temp dir first on PATH + require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath)) + + found, err := Which("mytool") + require.NoError(t, err) + + // Compare against the resolved path (handles /var -> /private/var on macOS) + expected, err := resolve(bin) + require.NoError(t, err) + assert.Equal(t, expected, found) + }) + + t.Run("resolves_symlinks", func(t *testing.T) { + // Symlink behavior varies on Windows (needs admin/developer mode). + if runtime.GOOS == "windows" { + t.Skip("symlink creation is restricted on Windows; test skipped") + } + dir := t.TempDir() + target := filepath.Join(dir, "realtool") + link := filepath.Join(dir, "alias") + + writeExec(t, target, "#!/bin/sh\necho real\n") + err := os.Symlink(target, link) + if err != nil { + t.Skipf("symlinks not supported in this environment: %v", err) + } + + require.NoError(t, os.Setenv("PATH", dir)) + + found, err := Which("alias") + require.NoError(t, err) + + expected, err := resolve(target) + require.NoError(t, err) + assert.Equal(t, expected, found, "Which should resolve through the symlink to the real path") + }) + + t.Run("direct_path_with_separator", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("executable bit semantics differ on Windows; test skipped") + } + dir := t.TempDir() + bin := filepath.Join(dir, "runme") + writeExec(t, bin, "#!/bin/sh\necho hi\n") + + found, err := Which(bin) + require.NoError(t, err) + + expected, err := resolve(bin) + require.NoError(t, err) + assert.Equal(t, expected, found) + }) + }) +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + err := os.WriteFile(path, []byte(content), 0o644) + require.NoError(t, err) +} + +func writeExec(t *testing.T, path, content string) { + t.Helper() + err := os.WriteFile(path, []byte(content), 0o755) + require.NoError(t, err) + + // On some systems, writing then chmod is safer/more explicit. + err = os.Chmod(path, 0o755) + require.NoError(t, err) + + // Sanity: ensure it's not a directory and has any execute bit set (for non-Windows cases we test) + info, err := os.Stat(path) + require.NoError(t, err) + require.False(t, info.IsDir()) + require.True(t, info.Mode()&0o111 != 0) +} + +func readFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + require.NoError(t, err) + // Normalize Windows newlines just in case, so substring checks are reliable. + return strings.ReplaceAll(string(b), "\r\n", "\n") +} diff --git a/resources/metadata/codeqlExecuteScan.yaml b/resources/metadata/codeqlExecuteScan.yaml index 4c1838c79..7d7be196a 100644 --- a/resources/metadata/codeqlExecuteScan.yaml +++ b/resources/metadata/codeqlExecuteScan.yaml @@ -247,6 +247,31 @@ spec: - STEPS - STAGES - PARAMETERS + - name: paths + type: string + description: | + List of file or directory patterns to include. + Each entry must be on its own line, e.g.: + src/** + lib/** + Note: This parameter is only applicable for interpreted languages. + scope: + - STEPS + - STAGES + - PARAMETERS + - name: pathsIgnore + type: string + pathsIgnore: + description: | + List of file or directory patterns to ignore. + Each entry must be on its own line, e.g.: + **/*.md + docs/** + Note: This parameter is only applicable for interpreted languages. + scope: + - STEPS + - STAGES + - PARAMETERS containers: - image: "" outputs: