1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-10-30 23:57:50 +02:00

feat(codeql): ignore paths and scan paths (#5477)

This commit is contained in:
Timur Akhmadiev
2025-09-12 19:37:30 +04:00
committed by GitHub
parent b042ba490e
commit 48582710bd
6 changed files with 655 additions and 0 deletions

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"os"
"path"
"path/filepath"
"strings"
@@ -136,11 +137,38 @@ func printCodeqlImageVersion() {
}
}
func prepareCodeQLConfigFile(config *codeqlExecuteScanOptions) error {
pathsToScan := codeql.ParsePaths(config.Paths)
pathsToIgnore := codeql.ParsePaths(config.PathsIgnore)
codeQLExecName := "codeql"
codeQLPath, err := codeql.Which(codeQLExecName)
if err != nil {
return fmt.Errorf("could not locate codeql executable %w", err)
}
location, fileName := path.Split(codeQLPath)
if fileName != codeQLExecName {
return fmt.Errorf("could not find codeql executable in path: %s", codeQLPath)
}
defaultConfigLocation := path.Join(location, "default-codeql-config.yml")
err = codeql.AppendCodeQLPaths(defaultConfigLocation, pathsToScan, pathsToIgnore)
if err != nil {
return fmt.Errorf("append paths and paths ignore to the default config: %w", err)
}
return nil
}
func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, utils codeqlExecuteScanUtils, influx *codeqlExecuteScanInflux) ([]piperutils.Path, error) {
printCodeqlImageVersion()
var reports []piperutils.Path
err := prepareCodeQLConfigFile(config)
if err != nil {
log.Entry().WithError(err).Error("failed to prepare codeql config file")
return reports, err
}
dbCreateCustomFlags := codeql.ParseCustomFlags(config.DatabaseCreateFlags)
isMultiLang, err := runDatabaseCreate(config, dbCreateCustomFlags, utils)
if err != nil {

View File

@@ -48,6 +48,8 @@ type codeqlExecuteScanOptions struct {
DatabaseAnalyzeFlags string `json:"databaseAnalyzeFlags,omitempty"`
CustomCommand string `json:"customCommand,omitempty"`
TransformQuerySuite string `json:"transformQuerySuite,omitempty"`
Paths string `json:"paths,omitempty"`
PathsIgnore string `json:"pathsIgnore,omitempty"`
}
type codeqlExecuteScanInflux struct {
@@ -306,6 +308,8 @@ func addCodeqlExecuteScanFlags(cmd *cobra.Command, stepConfig *codeqlExecuteScan
cmd.Flags().StringVar(&stepConfig.DatabaseAnalyzeFlags, "databaseAnalyzeFlags", os.Getenv("PIPER_databaseAnalyzeFlags"), "A space-separated string of flags for the 'codeql database analyze' command.")
cmd.Flags().StringVar(&stepConfig.CustomCommand, "customCommand", os.Getenv("PIPER_customCommand"), "A custom user-defined command to run between codeql analysis and results upload.")
cmd.Flags().StringVar(&stepConfig.TransformQuerySuite, "transformQuerySuite", os.Getenv("PIPER_transformQuerySuite"), "A transform string that will be applied to the querySuite using the sed command.")
cmd.Flags().StringVar(&stepConfig.Paths, "paths", os.Getenv("PIPER_paths"), "List of file or directory patterns to include.\nEach entry must be on its own line, e.g.:\n src/**\n lib/**\nNote: This parameter is only applicable for interpreted languages.\n")
cmd.Flags().StringVar(&stepConfig.PathsIgnore, "pathsIgnore", os.Getenv("PIPER_pathsIgnore"), "List of file or directory patterns to ignore.\nEach entry must be on its own line, e.g.:\n **/*.md\n docs/**\nNote: This parameter is only applicable for interpreted languages.\n")
cmd.MarkFlagRequired("buildTool")
}
@@ -580,6 +584,24 @@ func codeqlExecuteScanMetadata() config.StepData {
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_transformQuerySuite"),
},
{
Name: "paths",
ResourceRef: []config.ResourceReference{},
Scope: []string{"STEPS", "STAGES", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_paths"),
},
{
Name: "pathsIgnore",
ResourceRef: []config.ResourceReference{},
Scope: []string{"STEPS", "STAGES", "PARAMETERS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_pathsIgnore"),
},
},
},
Containers: []config.Container{

View File

@@ -7,11 +7,15 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/SAP/jenkins-library/pkg/codeql"
"github.com/SAP/jenkins-library/pkg/mock"
@@ -1077,3 +1081,148 @@ func TestRunCustomCommand(t *testing.T) {
assert.Error(t, err)
})
}
func Test_prepareCodeQLConfigFile(t *testing.T) {
t.Run("creates_or_updates_default_config_next_to_codeql_binary", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("requires POSIX exec bits for the fake binary")
}
dir := t.TempDir()
codeqlBin := filepath.Join(dir, "codeql")
writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n")
origPath := os.Getenv("PATH")
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
opts := &codeqlExecuteScanOptions{
Paths: " src \nlib/utils\n",
PathsIgnore: "vendor\n**/*.gen.go\n",
}
err := prepareCodeQLConfigFile(opts)
require.NoError(t, err)
qlPath, err := codeql.Which("codeql")
require.NoError(t, err)
loc, found := strings.CutSuffix(qlPath, "codeql")
require.True(t, found)
cfgPath := path.Join(loc, "default-codeql-config.yml")
b, err := os.ReadFile(cfgPath)
require.NoError(t, err)
out := normalizeNL(string(b))
// Assert it contains the expected lists (don’t depend on ordering)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- src")
assert.Contains(t, out, "- lib/utils")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- vendor")
// yaml.v3 single-quotes strings with '*' etc.
assert.Contains(t, out, "- '**/*.gen.go'")
})
t.Run("when_codeql_binary_missing_returns_error", func(t *testing.T) {
dir := t.TempDir()
origPath := os.Getenv("PATH")
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
// PATH without codeql
require.NoError(t, os.Setenv("PATH", dir))
opts := &codeqlExecuteScanOptions{
Paths: "a",
PathsIgnore: "b",
}
err := prepareCodeQLConfigFile(opts)
require.Error(t, err)
assert.Contains(t, err.Error(), "could not locate codeql executable")
})
t.Run("propagates_append_error_when_default_config_path_is_a_directory", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("requires POSIX exec bits and reliable directory perms")
}
dir := t.TempDir()
// Create fake codeql binary
codeqlBin := filepath.Join(dir, "codeql")
writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n")
defaultCfgDir := filepath.Join(dir, "default-codeql-config.yml")
require.NoError(t, os.Mkdir(defaultCfgDir, 0o755))
origPath := os.Getenv("PATH")
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
opts := &codeqlExecuteScanOptions{
Paths: "x",
PathsIgnore: "y",
}
err := prepareCodeQLConfigFile(opts)
require.Error(t, err)
assert.Contains(t, err.Error(), "append paths and paths ignore to the default config")
})
t.Run("no_changes_when_both_paths_empty_but_still_needs_binary", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("requires POSIX exec bits for the fake binary")
}
dir := t.TempDir()
codeqlBin := filepath.Join(dir, "codeql")
writeExec(t, codeqlBin, "#!/bin/sh\necho CodeQL\n")
origPath := os.Getenv("PATH")
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
cfgFile := filepath.Join(dir, "default-codeql-config.yml")
writeCodeQLFile(t, cfgFile, "")
opts := &codeqlExecuteScanOptions{
Paths: "",
PathsIgnore: "",
}
before := readCodeQLFile(t, cfgFile)
time.Sleep(10 * time.Millisecond) // avoid flakiness on very fast FS
err := prepareCodeQLConfigFile(opts)
require.NoError(t, err)
after := readCodeQLFile(t, cfgFile)
assert.Equal(t, before, after, "file should remain unchanged when nothing to write")
})
}
func writeExec(t *testing.T, path, content string) {
t.Helper()
require.NoError(t, os.WriteFile(path, []byte(content), 0o755))
require.NoError(t, os.Chmod(path, 0o755))
info, err := os.Stat(path)
require.NoError(t, err)
require.False(t, info.IsDir())
require.NotZero(t, info.Mode()&0o111, "should be executable")
}
func writeCodeQLFile(t *testing.T, path, content string) {
t.Helper()
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
}
func readCodeQLFile(t *testing.T, path string) string {
t.Helper()
b, err := os.ReadFile(path)
require.NoError(t, err)
return string(b)
}
func normalizeNL(s string) string {
return strings.ReplaceAll(s, "\r\n", "\n")
}

105
pkg/codeql/path.go Normal file
View File

@@ -0,0 +1,105 @@
package codeql
import (
"os"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
)
// AppendCodeQLPaths updates the CodeQL config YAML with new paths/paths-ignore.
func AppendCodeQLPaths(cfgPath string, scanPaths, ignorePaths []string) error {
if len(scanPaths) == 0 && len(ignorePaths) == 0 {
// if both paths are empty - do not touch anything
return nil
}
var cfg map[string]any
data, err := os.ReadFile(cfgPath)
if err != nil {
if !os.IsNotExist(err) {
return err
}
} else {
// file exists; unmarshal if non-empty
if len(data) > 0 {
if err = yaml.Unmarshal(data, &cfg); err != nil {
return err
}
}
}
if cfg == nil {
// start from empty config if file doesn't exist or empty
cfg = make(map[string]any)
}
if len(scanPaths) != 0 {
cfg["paths"] = scanPaths
}
if len(ignorePaths) != 0 {
cfg["paths-ignore"] = ignorePaths
}
out, err := yaml.Marshal(cfg)
if err != nil {
return err
}
return os.WriteFile(cfgPath, out, 0o644)
}
func ParsePaths(pathsStr string) []string {
var paths []string
patterns := strings.Split(pathsStr, "\n")
for _, p := range patterns {
p = strings.TrimSpace(p)
if p == "" {
continue
}
paths = append(paths, p)
}
return paths
}
// Which finds the first executable in PATH and resolves symlinks.
func Which(name string) (string, error) {
// If the name already has a path separator, check it directly.
if strings.ContainsRune(name, filepath.Separator) {
if isExecutable(name) {
return resolve(name)
}
return "", os.ErrNotExist
}
// Search in PATH
for _, dir := range filepath.SplitList(os.Getenv("PATH")) {
if dir == "" {
dir = "."
}
candidate := filepath.Join(dir, name)
if isExecutable(candidate) {
return resolve(candidate)
}
}
return "", os.ErrNotExist
}
func isExecutable(path string) bool {
info, err := os.Stat(path)
if err != nil {
return false
}
if info.IsDir() {
return false
}
// POSIX: check execute bit
return info.Mode()&0o111 != 0
}
func resolve(path string) (string, error) {
abs, err := filepath.Abs(path)
if err != nil {
return "", err
}
return filepath.EvalSymlinks(abs)
}

326
pkg/codeql/path_test.go Normal file
View File

@@ -0,0 +1,326 @@
package codeql
import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAppendCodeQLPaths(t *testing.T) {
t.Run("AppendCodeQLPaths", func(t *testing.T) {
t.Run("creates_file_if_not_exists_with_both_keys", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
err := AppendCodeQLPaths(cfgPath,
[]string{"src", "lib/utils"},
[]string{"vendor", "**/*.gen.go"},
)
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- src")
assert.Contains(t, out, "- lib/utils")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- vendor")
assert.Contains(t, out, "- '**/*.gen.go'") // YAML encoder uses single quotes for '*'
})
t.Run("creates_file_if_not_exists_with_only_paths", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
err := AppendCodeQLPaths(cfgPath, []string{"a", "b"}, nil)
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- a")
assert.Contains(t, out, "- b")
assert.NotContains(t, out, "paths-ignore:")
})
t.Run("creates_file_if_not_exists_with_only_paths_ignore", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
err := AppendCodeQLPaths(cfgPath, nil, []string{"vendor"})
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.NotContains(t, out, "paths:\n")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- vendor")
})
t.Run("appends_to_empty_yaml_creates_paths_and_paths_ignore", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
writeFile(t, cfgPath, "")
err := AppendCodeQLPaths(cfgPath,
[]string{"src", "lib/utils"},
[]string{"vendor", "**/*.gen.go"},
)
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- src")
assert.Contains(t, out, "- lib/utils")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- vendor")
assert.Contains(t, out, "- '**/*.gen.go'")
})
t.Run("overwrites_on_type_mismatch_and_preserves_other_keys", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
// Intentionally set scalar for paths to force overwrite; and keep an existing ignore.
initial := "paths: foo\npaths-ignore:\n - old-ignore\n"
writeFile(t, cfgPath, initial)
err := AppendCodeQLPaths(cfgPath, []string{"new-path"}, nil)
require.NoError(t, err)
out := readFile(t, cfgPath)
// 'paths' should now be a sequence containing only the new value
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- new-path")
assert.NotContains(t, out, "foo")
// paths-ignore should remain present (we didn't provide a new ignore list)
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- old-ignore")
})
t.Run("updates_only_paths_when_ignore_not_provided", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
initial := "paths:\n - old\npaths-ignore:\n - keep-ignore\n"
writeFile(t, cfgPath, initial)
err := AppendCodeQLPaths(cfgPath, []string{"new-a", "new-b"}, nil)
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- new-a")
assert.Contains(t, out, "- new-b")
assert.NotContains(t, out, "- old")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- keep-ignore")
})
t.Run("updates_only_paths_ignore_when_paths_not_provided", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
initial := "paths:\n - keep\npaths-ignore:\n - old-ignore\n"
writeFile(t, cfgPath, initial)
err := AppendCodeQLPaths(cfgPath, nil, []string{"new-ignore-1", "**/*.gen.go"})
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- keep")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- new-ignore-1")
assert.Contains(t, out, "- '**/*.gen.go'")
assert.NotContains(t, out, "- old-ignore")
})
t.Run("no_op_when_no_values_on_existing_file", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
initial := "paths:\n - keep\npaths-ignore:\n - keep-ignore\n"
writeFile(t, cfgPath, initial)
before := readFile(t, cfgPath)
// Touch mtime differences robustly across filesystems.
time.Sleep(10 * time.Millisecond)
err := AppendCodeQLPaths(cfgPath, nil, nil)
require.NoError(t, err)
after := readFile(t, cfgPath)
assert.Equal(t, before, after, "file content should be unchanged for no-op with nil slices")
})
t.Run("no_op_when_no_values_and_file_missing_does_not_create_file", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
err := AppendCodeQLPaths(cfgPath, []string{}, []string{})
require.NoError(t, err)
_, statErr := os.Stat(cfgPath)
require.True(t, os.IsNotExist(statErr), "file should not be created when both inputs are empty")
})
t.Run("works_with_existing_empty_file", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "codeql.yml")
writeFile(t, cfgPath, "")
err := AppendCodeQLPaths(cfgPath, []string{"p1"}, []string{"i1"})
require.NoError(t, err)
out := readFile(t, cfgPath)
assert.Contains(t, out, "paths:")
assert.Contains(t, out, "- p1")
assert.Contains(t, out, "paths-ignore:")
assert.Contains(t, out, "- i1")
})
})
}
func TestParsePaths(t *testing.T) {
t.Run("ParsePaths", func(t *testing.T) {
cases := []struct {
name string
in string
out []string
}{
{
name: "trims_and_skips_blanks",
in: " src \n\nlib\n \nvendor ",
out: []string{"src", "lib", "vendor"},
},
{
name: "handles_single_line",
in: "only/one",
out: []string{"only/one"},
},
{
name: "keeps_globs_and_spaces_inside_line",
in: "**/*.go\npath with space/\n./rel",
out: []string{"**/*.go", "path with space/", "./rel"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := ParsePaths(tc.in)
assert.Equal(t, tc.out, got)
})
}
})
}
func TestWhich(t *testing.T) {
t.Run("Which", func(t *testing.T) {
// Save and restore PATH for isolation.
origPath := os.Getenv("PATH")
t.Cleanup(func() { _ = os.Setenv("PATH", origPath) })
t.Run("nonexistent_returns_ErrNotExist", func(t *testing.T) {
_, err := Which("definitely-not-a-real-binary-name")
assert.ErrorIs(t, err, os.ErrNotExist)
})
t.Run("finds_executable_in_PATH", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("executable bit semantics differ on Windows; test skipped")
}
dir := t.TempDir()
bin := filepath.Join(dir, "mytool")
writeExec(t, bin, "#!/bin/sh\necho ok\n")
// Put our temp dir first on PATH
require.NoError(t, os.Setenv("PATH", dir+string(os.PathListSeparator)+origPath))
found, err := Which("mytool")
require.NoError(t, err)
// Compare against the resolved path (handles /var -> /private/var on macOS)
expected, err := resolve(bin)
require.NoError(t, err)
assert.Equal(t, expected, found)
})
t.Run("resolves_symlinks", func(t *testing.T) {
// Symlink behavior varies on Windows (needs admin/developer mode).
if runtime.GOOS == "windows" {
t.Skip("symlink creation is restricted on Windows; test skipped")
}
dir := t.TempDir()
target := filepath.Join(dir, "realtool")
link := filepath.Join(dir, "alias")
writeExec(t, target, "#!/bin/sh\necho real\n")
err := os.Symlink(target, link)
if err != nil {
t.Skipf("symlinks not supported in this environment: %v", err)
}
require.NoError(t, os.Setenv("PATH", dir))
found, err := Which("alias")
require.NoError(t, err)
expected, err := resolve(target)
require.NoError(t, err)
assert.Equal(t, expected, found, "Which should resolve through the symlink to the real path")
})
t.Run("direct_path_with_separator", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("executable bit semantics differ on Windows; test skipped")
}
dir := t.TempDir()
bin := filepath.Join(dir, "runme")
writeExec(t, bin, "#!/bin/sh\necho hi\n")
found, err := Which(bin)
require.NoError(t, err)
expected, err := resolve(bin)
require.NoError(t, err)
assert.Equal(t, expected, found)
})
})
}
func writeFile(t *testing.T, path, content string) {
t.Helper()
err := os.WriteFile(path, []byte(content), 0o644)
require.NoError(t, err)
}
func writeExec(t *testing.T, path, content string) {
t.Helper()
err := os.WriteFile(path, []byte(content), 0o755)
require.NoError(t, err)
// On some systems, writing then chmod is safer/more explicit.
err = os.Chmod(path, 0o755)
require.NoError(t, err)
// Sanity: ensure it's not a directory and has any execute bit set (for non-Windows cases we test)
info, err := os.Stat(path)
require.NoError(t, err)
require.False(t, info.IsDir())
require.True(t, info.Mode()&0o111 != 0)
}
func readFile(t *testing.T, path string) string {
t.Helper()
b, err := os.ReadFile(path)
require.NoError(t, err)
// Normalize Windows newlines just in case, so substring checks are reliable.
return strings.ReplaceAll(string(b), "\r\n", "\n")
}

View File

@@ -247,6 +247,31 @@ spec:
- STEPS
- STAGES
- PARAMETERS
- name: paths
type: string
description: |
List of file or directory patterns to include.
Each entry must be on its own line, e.g.:
src/**
lib/**
Note: This parameter is only applicable for interpreted languages.
scope:
- STEPS
- STAGES
- PARAMETERS
- name: pathsIgnore
type: string
pathsIgnore:
description: |
List of file or directory patterns to ignore.
Each entry must be on its own line, e.g.:
**/*.md
docs/**
Note: This parameter is only applicable for interpreted languages.
scope:
- STEPS
- STAGES
- PARAMETERS
containers:
- image: ""
outputs: