feat(codeql): new codeql piper step (#3765)

* codeql piper step

cmd/codeqlExecuteScan.go

@@ -0,0 +1,231 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+
+	"github.com/SAP/jenkins-library/pkg/command"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/orchestrator"
+	"github.com/SAP/jenkins-library/pkg/piperutils"
+	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/pkg/errors"
+)
+
+type codeqlExecuteScanUtils interface {
+	command.ExecRunner
+
+	FileExists(filename string) (bool, error)
+}
+
+type RepoInfo struct {
+	serverUrl string
+	repo      string
+	commitId  string
+	ref       string
+}
+
+type codeqlExecuteScanUtilsBundle struct {
+	*command.Command
+	*piperutils.Files
+}
+
+func newCodeqlExecuteScanUtils() codeqlExecuteScanUtils {
+	utils := codeqlExecuteScanUtilsBundle{
+		Command: &command.Command{},
+		Files:   &piperutils.Files{},
+	}
+
+	utils.Stdout(log.Writer())
+	utils.Stderr(log.Writer())
+	return &utils
+}
+
+func codeqlExecuteScan(config codeqlExecuteScanOptions, telemetryData *telemetry.CustomData) {
+
+	utils := newCodeqlExecuteScanUtils()
+
+	err := runCodeqlExecuteScan(&config, telemetryData, utils)
+	if err != nil {
+		log.Entry().WithError(err).Fatal("Codeql scan failed")
+	}
+}
+
+func codeqlQuery(cmd []string, codeqlQuery string) []string {
+	if len(codeqlQuery) > 0 {
+		cmd = append(cmd, codeqlQuery)
+	}
+
+	return cmd
+}
+
+func execute(utils codeqlExecuteScanUtils, cmd []string, isVerbose bool) error {
+	if isVerbose {
+		cmd = append(cmd, "-v")
+	}
+	return utils.RunExecutable("codeql", cmd...)
+}
+
+func getLangFromBuildTool(buildTool string) string {
+	switch buildTool {
+	case "maven":
+		return "java"
+	case "pip":
+		return "python"
+	case "npm":
+		return "javascript"
+	case "yarn":
+		return "javascript"
+	case "golang":
+		return "go"
+	default:
+		return ""
+	}
+}
+
+func getGitRepoInfo(repoUri string, repoInfo *RepoInfo) error {
+	if repoUri == "" {
+		return errors.New("repository param is not set or it cannot be auto populated")
+	}
+
+	pat := regexp.MustCompile(`^(https|git)(:\/\/|@)([^\/:]+)[\/:]([^\/:]+\/[^.]+)(.git)*$`)
+	matches := pat.FindAllStringSubmatch(repoUri, -1)
+	if len(matches) > 0 {
+		match := matches[0]
+		repoInfo.serverUrl = "https://" + match[3]
+		repoInfo.repo = match[4]
+		return nil
+	}
+
+	return fmt.Errorf("Invalid repository %s", repoUri)
+}
+
+func uploadResults(config *codeqlExecuteScanOptions, utils codeqlExecuteScanUtils) error {
+	if config.UploadResults {
+		if len(config.GithubToken) == 0 {
+			return errors.New("failed running upload-results as github token was not specified")
+		}
+
+		var repoInfo RepoInfo
+		err := getGitRepoInfo(config.Repository, &repoInfo)
+		if err != nil {
+			log.Entry().Error(err)
+		}
+		repoInfo.ref = config.AnalyzedRef
+		repoInfo.commitId = config.CommitID
+
+		provider, err := orchestrator.NewOrchestratorSpecificConfigProvider()
+		if err != nil {
+			log.Entry().Error(err)
+		} else {
+			if repoInfo.ref == "" {
+				repoInfo.ref = provider.GetReference()
+			}
+
+			if repoInfo.commitId == "" {
+				repoInfo.commitId = provider.GetCommit()
+			}
+
+			if repoInfo.serverUrl == "" {
+				err = getGitRepoInfo(provider.GetRepoURL(), &repoInfo)
+				if err != nil {
+					log.Entry().Error(err)
+				}
+			}
+		}
+
+		cmd := []string{"github", "upload-results", "--sarif=" + fmt.Sprintf("%vtarget/codeqlReport.sarif", config.ModulePath), "-a=" + config.GithubToken}
+
+		if repoInfo.commitId != "" {
+			cmd = append(cmd, "--commit="+repoInfo.commitId)
+		}
+
+		if repoInfo.serverUrl != "" {
+			cmd = append(cmd, "--github-url="+repoInfo.serverUrl)
+		}
+
+		if repoInfo.repo != "" {
+			cmd = append(cmd, "--repository="+repoInfo.repo)
+		}
+
+		if repoInfo.ref != "" {
+			cmd = append(cmd, "--ref="+repoInfo.ref)
+		}
+
+		//if no git pramas are passed(commitId, reference, serverUrl, repository), then codeql tries to auto populate it based on git information of the checkout repository.
+		//It also depends on the orchestrator. Some orchestrator keep git information and some not.
+		err = execute(utils, cmd, GeneralConfig.Verbose)
+		if err != nil {
+			log.Entry().Error("failed to upload sarif results")
+			return err
+		}
+	}
+
+	return nil
+}
+
+func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, utils codeqlExecuteScanUtils) error {
+	var reports []piperutils.Path
+	cmd := []string{"database", "create", "db", "--overwrite", "--source-root", config.ModulePath}
+
+	language := getLangFromBuildTool(config.BuildTool)
+
+	if len(language) == 0 && len(config.Language) == 0 {
+		if config.BuildTool == "custom" {
+			return fmt.Errorf("as the buildTool is custom. please atleast specify the language parameter")
+		} else {
+			return fmt.Errorf("the step could not recognize the specified buildTool %s. please specify valid buildtool", config.BuildTool)
+		}
+	}
+
+	cmd = append(cmd, "--language="+language)
+	if len(config.Language) > 0 {
+		cmd = append(cmd, "--language="+config.Language)
+	}
+
+	//codeql has an autobuilder which tries to build the project based on specified programming language
+	if len(config.BuildCommand) > 0 {
+		cmd = append(cmd, "--command="+config.BuildCommand)
+	}
+
+	err := execute(utils, cmd, GeneralConfig.Verbose)
+	if err != nil {
+		log.Entry().Error("failed running command codeql database create")
+		return err
+	}
+
+	os.MkdirAll(fmt.Sprintf("%vtarget", config.ModulePath), os.ModePerm)
+
+	cmd = nil
+	cmd = append(cmd, "database", "analyze", "--format=sarif-latest", fmt.Sprintf("--output=%vtarget/codeqlReport.sarif", config.ModulePath), "db")
+	cmd = codeqlQuery(cmd, config.QuerySuite)
+	err = execute(utils, cmd, GeneralConfig.Verbose)
+	if err != nil {
+		log.Entry().Error("failed running command codeql database analyze for sarif generation")
+		return err
+	}
+
+	reports = append(reports, piperutils.Path{Target: fmt.Sprintf("%vtarget/codeqlReport.sarif", config.ModulePath)})
+
+	cmd = nil
+	cmd = append(cmd, "database", "analyze", "--format=csv", fmt.Sprintf("--output=%vtarget/codeqlReport.csv", config.ModulePath), "db")
+	cmd = codeqlQuery(cmd, config.QuerySuite)
+	err = execute(utils, cmd, GeneralConfig.Verbose)
+	if err != nil {
+		log.Entry().Error("failed running command codeql database analyze for csv generation")
+		return err
+	}
+
+	reports = append(reports, piperutils.Path{Target: fmt.Sprintf("%vtarget/codeqlReport.csv", config.ModulePath)})
+
+	piperutils.PersistReportsAndLinks("codeqlExecuteScan", "./", reports, nil)
+
+	err = uploadResults(config, utils)
+	if err != nil {
+		log.Entry().Error("failed to upload results")
+		return err
+	}
+
+	return nil
+}

cmd/codeqlExecuteScan_generated.go

@@ -0,0 +1,335 @@
+// Code generated by piper's step-generator. DO NOT EDIT.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/SAP/jenkins-library/pkg/config"
+	"github.com/SAP/jenkins-library/pkg/gcs"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/splunk"
+	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/SAP/jenkins-library/pkg/validation"
+	"github.com/bmatcuk/doublestar"
+	"github.com/spf13/cobra"
+)
+
+type codeqlExecuteScanOptions struct {
+	GithubToken   string `json:"githubToken,omitempty"`
+	BuildTool     string `json:"buildTool,omitempty" validate:"possible-values=custom maven golang npm pip yarn"`
+	BuildCommand  string `json:"buildCommand,omitempty"`
+	Language      string `json:"language,omitempty"`
+	ModulePath    string `json:"modulePath,omitempty"`
+	QuerySuite    string `json:"querySuite,omitempty"`
+	UploadResults bool   `json:"uploadResults,omitempty"`
+	AnalyzedRef   string `json:"analyzedRef,omitempty"`
+	Repository    string `json:"repository,omitempty"`
+	CommitID      string `json:"commitId,omitempty"`
+}
+
+type codeqlExecuteScanReports struct {
+}
+
+func (p *codeqlExecuteScanReports) persist(stepConfig codeqlExecuteScanOptions, gcpJsonKeyFilePath string, gcsBucketId string, gcsFolderPath string, gcsSubFolder string) {
+	if gcsBucketId == "" {
+		log.Entry().Info("persisting reports to GCS is disabled, because gcsBucketId is empty")
+		return
+	}
+	log.Entry().Info("Uploading reports to Google Cloud Storage...")
+	content := []gcs.ReportOutputParam{
+		{FilePattern: "**/*.csv", ParamRef: "", StepResultType: "codeql"},
+		{FilePattern: "**/*.sarif", ParamRef: "", StepResultType: "codeql"},
+	}
+	envVars := []gcs.EnvVar{
+		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: gcpJsonKeyFilePath, Modified: false},
+	}
+	gcsClient, err := gcs.NewClient(gcs.WithEnvVars(envVars))
+	if err != nil {
+		log.Entry().Errorf("creation of GCS client failed: %v", err)
+		return
+	}
+	defer gcsClient.Close()
+	structVal := reflect.ValueOf(&stepConfig).Elem()
+	inputParameters := map[string]string{}
+	for i := 0; i < structVal.NumField(); i++ {
+		field := structVal.Type().Field(i)
+		if field.Type.String() == "string" {
+			paramName := strings.Split(field.Tag.Get("json"), ",")
+			paramValue, _ := structVal.Field(i).Interface().(string)
+			inputParameters[paramName[0]] = paramValue
+		}
+	}
+	if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, gcsFolderPath, gcsBucketId, gcsSubFolder, doublestar.Glob, os.Stat); err != nil {
+		log.Entry().Errorf("failed to persist reports: %v", err)
+	}
+}
+
+// CodeqlExecuteScanCommand This step executes a codeql scan on the specified project to perform static code analysis and check the source code for security flaws.
+func CodeqlExecuteScanCommand() *cobra.Command {
+	const STEP_NAME = "codeqlExecuteScan"
+
+	metadata := codeqlExecuteScanMetadata()
+	var stepConfig codeqlExecuteScanOptions
+	var startTime time.Time
+	var reports codeqlExecuteScanReports
+	var logCollector *log.CollectorHook
+	var splunkClient *splunk.Splunk
+	telemetryClient := &telemetry.Telemetry{}
+
+	var createCodeqlExecuteScanCmd = &cobra.Command{
+		Use:   STEP_NAME,
+		Short: "This step executes a codeql scan on the specified project to perform static code analysis and check the source code for security flaws.",
+		Long: `This step executes a codeql scan on the specified project to perform static code analysis and check the source code for security flaws.
+
+The codeql step triggers a scan locally on your orchestrator (e.g. Jenkins) within a docker container so finally you have to supply a docker image with codeql
+and Java plus Maven.`,
+		PreRunE: func(cmd *cobra.Command, _ []string) error {
+			startTime = time.Now()
+			log.SetStepName(STEP_NAME)
+			log.SetVerbose(GeneralConfig.Verbose)
+
+			GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens)
+
+			path, _ := os.Getwd()
+			fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path}
+			log.RegisterHook(fatalHook)
+
+			err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile)
+			if err != nil {
+				log.SetErrorCategory(log.ErrorConfiguration)
+				return err
+			}
+			log.RegisterSecret(stepConfig.GithubToken)
+
+			if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 {
+				sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID)
+				log.RegisterHook(&sentryHook)
+			}
+
+			if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+				splunkClient = &splunk.Splunk{}
+				logCollector = &log.CollectorHook{CorrelationID: GeneralConfig.CorrelationID}
+				log.RegisterHook(logCollector)
+			}
+
+			if err = log.RegisterANSHookIfConfigured(GeneralConfig.CorrelationID); err != nil {
+				log.Entry().WithError(err).Warn("failed to set up SAP Alert Notification Service log hook")
+			}
+
+			validation, err := validation.New(validation.WithJSONNamesForStructFields(), validation.WithPredefinedErrorMessages())
+			if err != nil {
+				return err
+			}
+			if err = validation.ValidateStruct(stepConfig); err != nil {
+				log.SetErrorCategory(log.ErrorConfiguration)
+				return err
+			}
+
+			return nil
+		},
+		Run: func(_ *cobra.Command, _ []string) {
+			stepTelemetryData := telemetry.CustomData{}
+			stepTelemetryData.ErrorCode = "1"
+			handler := func() {
+				reports.persist(stepConfig, GeneralConfig.GCPJsonKeyFilePath, GeneralConfig.GCSBucketId, GeneralConfig.GCSFolderPath, GeneralConfig.GCSSubFolder)
+				config.RemoveVaultSecretFiles()
+				stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds())
+				stepTelemetryData.ErrorCategory = log.GetErrorCategory().String()
+				stepTelemetryData.PiperCommitHash = GitCommit
+				telemetryClient.SetData(&stepTelemetryData)
+				telemetryClient.Send()
+				if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+					splunkClient.Send(telemetryClient.GetData(), logCollector)
+				}
+			}
+			log.DeferExitHandler(handler)
+			defer handler()
+			telemetryClient.Initialize(GeneralConfig.NoTelemetry, STEP_NAME)
+			if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+				splunkClient.Initialize(GeneralConfig.CorrelationID,
+					GeneralConfig.HookConfig.SplunkConfig.Dsn,
+					GeneralConfig.HookConfig.SplunkConfig.Token,
+					GeneralConfig.HookConfig.SplunkConfig.Index,
+					GeneralConfig.HookConfig.SplunkConfig.SendLogs)
+			}
+			codeqlExecuteScan(stepConfig, &stepTelemetryData)
+			stepTelemetryData.ErrorCode = "0"
+			log.Entry().Info("SUCCESS")
+		},
+	}
+
+	addCodeqlExecuteScanFlags(createCodeqlExecuteScanCmd, &stepConfig)
+	return createCodeqlExecuteScanCmd
+}
+
+func addCodeqlExecuteScanFlags(cmd *cobra.Command, stepConfig *codeqlExecuteScanOptions) {
+	cmd.Flags().StringVar(&stepConfig.GithubToken, "githubToken", os.Getenv("PIPER_githubToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line")
+	cmd.Flags().StringVar(&stepConfig.BuildTool, "buildTool", `maven`, "Defines the build tool which is used for building the project.")
+	cmd.Flags().StringVar(&stepConfig.BuildCommand, "buildCommand", os.Getenv("PIPER_buildCommand"), "Command to build the project")
+	cmd.Flags().StringVar(&stepConfig.Language, "language", os.Getenv("PIPER_language"), "The programming language used to analyze.")
+	cmd.Flags().StringVar(&stepConfig.ModulePath, "modulePath", `./`, "Allows providing the path for the module to scan")
+	cmd.Flags().StringVar(&stepConfig.QuerySuite, "querySuite", os.Getenv("PIPER_querySuite"), "The name of a CodeQL query suite. If omitted, the default query suite for the language of the database being analyzed will be used.")
+	cmd.Flags().BoolVar(&stepConfig.UploadResults, "uploadResults", false, "Allows you to upload codeql SARIF results to your github project. You will need to set githubToken for this.")
+	cmd.Flags().StringVar(&stepConfig.AnalyzedRef, "analyzedRef", os.Getenv("PIPER_analyzedRef"), "Name of the ref that was analyzed.")
+	cmd.Flags().StringVar(&stepConfig.Repository, "repository", os.Getenv("PIPER_repository"), "URL of the GitHub instance")
+	cmd.Flags().StringVar(&stepConfig.CommitID, "commitId", os.Getenv("PIPER_commitId"), "SHA of commit that was analyzed.")
+
+	cmd.MarkFlagRequired("buildTool")
+}
+
+// retrieve step metadata
+func codeqlExecuteScanMetadata() config.StepData {
+	var theMetaData = config.StepData{
+		Metadata: config.StepMetadata{
+			Name:        "codeqlExecuteScan",
+			Aliases:     []config.Alias{},
+			Description: "This step executes a codeql scan on the specified project to perform static code analysis and check the source code for security flaws.",
+		},
+		Spec: config.StepSpec{
+			Inputs: config.StepInputs{
+				Secrets: []config.StepSecrets{
+					{Name: "githubTokenCredentialsId", Description: "Jenkins 'Secret text' credentials ID containing token to authenticate to GitHub.", Type: "jenkins"},
+				},
+				Parameters: []config.StepParameters{
+					{
+						Name: "githubToken",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name: "githubTokenCredentialsId",
+								Type: "secret",
+							},
+
+							{
+								Name:    "githubVaultSecretName",
+								Type:    "vaultSecret",
+								Default: "github",
+							},
+						},
+						Scope:     []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{{Name: "access_token"}},
+						Default:   os.Getenv("PIPER_githubToken"),
+					},
+					{
+						Name:        "buildTool",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   true,
+						Aliases:     []config.Alias{},
+						Default:     `maven`,
+					},
+					{
+						Name:        "buildCommand",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_buildCommand"),
+					},
+					{
+						Name:        "language",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_language"),
+					},
+					{
+						Name:        "modulePath",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     `./`,
+					},
+					{
+						Name:        "querySuite",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_querySuite"),
+					},
+					{
+						Name:        "uploadResults",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "bool",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     false,
+					},
+					{
+						Name: "analyzedRef",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name:  "commonPipelineEnvironment",
+								Param: "git/ref",
+							},
+						},
+						Scope:     []string{},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{},
+						Default:   os.Getenv("PIPER_analyzedRef"),
+					},
+					{
+						Name: "repository",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name:  "commonPipelineEnvironment",
+								Param: "git/httpsUrl",
+							},
+						},
+						Scope:     []string{},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{{Name: "githubRepo"}},
+						Default:   os.Getenv("PIPER_repository"),
+					},
+					{
+						Name: "commitId",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name:  "commonPipelineEnvironment",
+								Param: "git/commitId",
+							},
+						},
+						Scope:     []string{},
+						Type:      "string",
+						Mandatory: false,
+						Aliases:   []config.Alias{},
+						Default:   os.Getenv("PIPER_commitId"),
+					},
+				},
+			},
+			Containers: []config.Container{
+				{},
+			},
+			Outputs: config.StepOutputs{
+				Resources: []config.StepResources{
+					{
+						Name: "reports",
+						Type: "reports",
+						Parameters: []map[string]interface{}{
+							{"filePattern": "**/*.csv", "type": "codeql"},
+							{"filePattern": "**/*.sarif", "type": "codeql"},
+						},
+					},
+				},
+			},
+		},
+	}
+	return theMetaData
+}

cmd/codeqlExecuteScan_generated_test.go

@@ -0,0 +1,17 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCodeqlExecuteScanCommand(t *testing.T) {
+	t.Parallel()
+
+	testCmd := CodeqlExecuteScanCommand()
+
+	// only high level testing performed - details are tested in step generation procedure
+	assert.Equal(t, "codeqlExecuteScan", testCmd.Use, "command name incorrect")
+
+}

cmd/codeqlExecuteScan_test.go

@@ -0,0 +1,85 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/SAP/jenkins-library/pkg/mock"
+	"github.com/stretchr/testify/assert"
+)
+
+type codeqlExecuteScanMockUtils struct {
+	*mock.ExecMockRunner
+	*mock.FilesMock
+}
+
+func newCodeqlExecuteScanTestsUtils() codeqlExecuteScanMockUtils {
+	utils := codeqlExecuteScanMockUtils{
+		ExecMockRunner: &mock.ExecMockRunner{},
+		FilesMock:      &mock.FilesMock{},
+	}
+	return utils
+}
+
+func TestRunCodeqlExecuteScan(t *testing.T) {
+
+	t.Run("Valid CodeqlExecuteScan", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{BuildTool: "maven", ModulePath: "./"}
+		assert.Equal(t, nil, runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils()))
+	})
+
+	t.Run("No auth token passed on upload results", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{BuildTool: "maven", UploadResults: true, ModulePath: "./"}
+		assert.Error(t, runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils()))
+	})
+
+	t.Run("Upload results with token", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{BuildTool: "maven", ModulePath: "./", UploadResults: true, GithubToken: "test"}
+		assert.Equal(t, nil, runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils()))
+	})
+
+	t.Run("Custom buildtool", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{BuildTool: "custom", Language: "javascript", ModulePath: "./", GithubToken: "test"}
+		assert.Equal(t, nil, runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils()))
+	})
+
+	t.Run("Custom buildtool but no language specified", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{BuildTool: "custom", ModulePath: "./", GithubToken: "test"}
+		assert.Error(t, runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils()))
+	})
+
+	t.Run("Invalid buildtool and no language specified", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{BuildTool: "test", ModulePath: "./", GithubToken: "test"}
+		assert.Error(t, runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils()))
+	})
+
+	t.Run("Invalid buildtool but language specified", func(t *testing.T) {
+		config := codeqlExecuteScanOptions{BuildTool: "test", Language: "javascript", ModulePath: "./", GithubToken: "test"}
+		assert.Equal(t, nil, runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils()))
+	})
+}
+
+func TestGetGitRepoInfo(t *testing.T) {
+	t.Run("Valid URL1", func(t *testing.T) {
+		var repoInfo RepoInfo
+		getGitRepoInfo("https://github.hello.test/Testing/fortify.git", &repoInfo)
+		assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl)
+		assert.Equal(t, "Testing/fortify", repoInfo.repo)
+	})
+
+	t.Run("Valid URL2", func(t *testing.T) {
+		var repoInfo RepoInfo
+		getGitRepoInfo("https://github.hello.test/Testing/fortify", &repoInfo)
+		assert.Equal(t, "https://github.hello.test", repoInfo.serverUrl)
+		assert.Equal(t, "Testing/fortify", repoInfo.repo)
+	})
+
+	t.Run("Invalid URL as no org/owner passed", func(t *testing.T) {
+		var repoInfo RepoInfo
+		assert.Error(t, getGitRepoInfo("https://github.com/fortify", &repoInfo))
+	})
+
+	t.Run("Invalid URL as no protocol passed", func(t *testing.T) {
+		var repoInfo RepoInfo
+		assert.Error(t, getGitRepoInfo("github.hello.test/Testing/fortify", &repoInfo))
+	})
+}

cmd/metadata_generated.go

@@ -45,6 +45,7 @@ func GetAllStepMetadata() map[string]config.StepData {
 		"cloudFoundryDeleteSpace":                   cloudFoundryDeleteSpaceMetadata(),
 		"cloudFoundryDeploy":                        cloudFoundryDeployMetadata(),
 		"cnbBuild":                                  cnbBuildMetadata(),
+		"codeqlExecuteScan":                         codeqlExecuteScanMetadata(),
 		"containerExecuteStructureTests":            containerExecuteStructureTestsMetadata(),
 		"containerSaveImage":                        containerSaveImageMetadata(),
 		"detectExecuteScan":                         detectExecuteScanMetadata(),

cmd/piper.go

@@ -112,6 +112,7 @@ func Execute() {
 	rootCmd.AddCommand(AbapEnvironmentCreateSystemCommand())
 	rootCmd.AddCommand(CheckmarxExecuteScanCommand())
 	rootCmd.AddCommand(FortifyExecuteScanCommand())
+	rootCmd.AddCommand(CodeqlExecuteScanCommand())
 	rootCmd.AddCommand(MtaBuildCommand())
 	rootCmd.AddCommand(ProtecodeExecuteScanCommand())
 	rootCmd.AddCommand(MavenExecuteCommand())

documentation/docs/steps/codeqlExecuteScan.md

@@ -0,0 +1,7 @@
+# ${docGenStepName}
+
+## ${docGenDescription}
+
+## ${docGenParameters}
+
+## ${docGenConfiguration}

documentation/mkdocs.yml

@@ -89,6 +89,7 @@ nav:
         - cloudFoundryDeleteService: steps/cloudFoundryDeleteService.md
         - cloudFoundryDeploy: steps/cloudFoundryDeploy.md
         - cnbBuild: steps/cnbBuild.md
+        - codeqlExecuteScan: steps/codeqlExecuteScan.md
         - commonPipelineEnvironment: steps/commonPipelineEnvironment.md
         - containerExecuteStructureTests: steps/containerExecuteStructureTests.md
         - containerPushToRegistry: steps/containerPushToRegistry.md

pkg/orchestrator/azureDevOps.go

@@ -1,13 +1,14 @@
 package orchestrator
 
 import (
-	piperHttp "github.com/SAP/jenkins-library/pkg/http"
-	"github.com/SAP/jenkins-library/pkg/log"
 	"io/ioutil"
 	"os"
 	"strconv"
 	"strings"
 	"time"
+
+	piperHttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/SAP/jenkins-library/pkg/log"
 )
 
 type AzureDevOpsConfigProvider struct {
@@ -192,6 +193,11 @@ func (a *AzureDevOpsConfigProvider) GetBranch() string {
 	return strings.TrimPrefix(tmp, "refs/heads/")
 }
 
+// GetReference return the git reference
+func (a *AzureDevOpsConfigProvider) GetReference() string {
+	return getEnv("BUILD_SOURCEBRANCH", "n/a")
+}
+
 // GetBuildURL returns the builds URL e.g. https://dev.azure.com/fabrikamfiber/your-repo-name/_build/results?buildId=1234
 func (a *AzureDevOpsConfigProvider) GetBuildURL() string {
 	return os.Getenv("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI") + os.Getenv("SYSTEM_TEAMPROJECT") + "/" + os.Getenv("SYSTEM_DEFINITIONNAME") + "/_build/results?buildId=" + a.getAzureBuildID()

pkg/orchestrator/azureDevOps_test.go

@@ -2,14 +2,15 @@ package orchestrator
 
 import (
 	"fmt"
-	piperhttp "github.com/SAP/jenkins-library/pkg/http"
-	"github.com/jarcoal/httpmock"
-	"github.com/pkg/errors"
 	"net/http"
 	"os"
 	"testing"
 	"time"
 
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/jarcoal/httpmock"
+	"github.com/pkg/errors"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -18,7 +19,7 @@ func TestAzure(t *testing.T) {
 		defer resetEnv(os.Environ())
 		os.Clearenv()
 		os.Setenv("AZURE_HTTP_USER_AGENT", "FOO BAR BAZ")
-		os.Setenv("BUILD_SOURCEBRANCH", "feat/test-azure")
+		os.Setenv("BUILD_SOURCEBRANCH", "refs/heads/feat/test-azure")
 		os.Setenv("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI", "https://pogo.sap/")
 		os.Setenv("SYSTEM_TEAMPROJECT", "foo")
 		os.Setenv("BUILD_BUILDID", "42")
@@ -30,6 +31,7 @@ func TestAzure(t *testing.T) {
 
 		assert.False(t, p.IsPullRequest())
 		assert.Equal(t, "feat/test-azure", p.GetBranch())
+		assert.Equal(t, "refs/heads/feat/test-azure", p.GetReference())
 		assert.Equal(t, "https://pogo.sap/foo/bar/_build/results?buildId=42", p.GetBuildURL())
 		assert.Equal(t, "abcdef42713", p.GetCommit())
 		assert.Equal(t, "github.com/foo/bar", p.GetRepoURL())

pkg/orchestrator/gitHubActions.go

@@ -1,10 +1,11 @@
 package orchestrator
 
 import (
-	"github.com/SAP/jenkins-library/pkg/log"
 	"os"
 	"strings"
 	"time"
+
+	"github.com/SAP/jenkins-library/pkg/log"
 )
 
 type GitHubActionsConfigProvider struct{}
@@ -53,6 +54,10 @@ func (g *GitHubActionsConfigProvider) GetBranch() string {
 	return strings.TrimPrefix(getEnv("GITHUB_REF", "n/a"), "refs/heads/")
 }
 
+func (g *GitHubActionsConfigProvider) GetReference() string {
+	return getEnv("GITHUB_REF", "n/a")
+}
+
 func (g *GitHubActionsConfigProvider) GetBuildURL() string {
 	return g.GetRepoURL() + "/actions/runs/" + getEnv("GITHUB_RUN_ID", "n/a")
 }

pkg/orchestrator/gitHubActions_test.go

@@ -24,6 +24,7 @@ func TestGitHubActions(t *testing.T) {
 		assert.False(t, p.IsPullRequest())
 		assert.Equal(t, "github.com/foo/bar/actions/runs/42", p.GetBuildURL())
 		assert.Equal(t, "feat/test-gh-actions", p.GetBranch())
+		assert.Equal(t, "refs/heads/feat/test-gh-actions", p.GetReference())
 		assert.Equal(t, "abcdef42713", p.GetCommit())
 		assert.Equal(t, "github.com/foo/bar", p.GetRepoURL())
 		assert.Equal(t, "GitHubActions", p.OrchestratorType())

pkg/orchestrator/jenkins.go

@@ -2,12 +2,14 @@ package orchestrator
 
 import (
 	"encoding/json"
+	"io/ioutil"
+	"strings"
+	"time"
+
 	"github.com/Jeffail/gabs/v2"
 	piperHttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/pkg/errors"
-	"io/ioutil"
-	"time"
 )
 
 type JenkinsConfigProvider struct {
@@ -196,6 +198,18 @@ func (j *JenkinsConfigProvider) GetBranch() string {
 	return getEnv("BRANCH_NAME", "n/a")
 }
 
+// GetReference returns the git reference, only works with the git plugin enabled
+func (j *JenkinsConfigProvider) GetReference() string {
+	ref := getEnv("BRANCH_NAME", "n/a")
+	if ref == "n/a" {
+		return ref
+	} else if strings.Contains(ref, "PR") {
+		return "refs/pull/" + strings.Split(ref, "-")[1] + "/head"
+	} else {
+		return "refs/heads/" + ref
+	}
+}
+
 // GetBuildURL returns the build url, e.g. https://jaas.url/job/foo/job/bar/job/main/1234/
 func (j *JenkinsConfigProvider) GetBuildURL() string {
 	return getEnv("BUILD_URL", "n/a")

pkg/orchestrator/jenkins_test.go

@@ -3,15 +3,17 @@ package orchestrator
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/pkg/errors"
 	"os"
 	"testing"
 	"time"
 
+	"github.com/pkg/errors"
+
+	"net/http"
+
 	piperhttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/jarcoal/httpmock"
 	"github.com/stretchr/testify/assert"
-	"net/http"
 )
 
 func TestJenkins(t *testing.T) {
@@ -29,6 +31,7 @@ func TestJenkins(t *testing.T) {
 		assert.False(t, p.IsPullRequest())
 		assert.Equal(t, "https://jaas.url/job/foo/job/bar/job/main/1234/", p.GetBuildURL())
 		assert.Equal(t, "main", p.GetBranch())
+		assert.Equal(t, "refs/heads/main", p.GetReference())
 		assert.Equal(t, "abcdef42713", p.GetCommit())
 		assert.Equal(t, "github.com/foo/bar", p.GetRepoURL())
 		assert.Equal(t, "Jenkins", p.OrchestratorType())
@@ -46,6 +49,7 @@ func TestJenkins(t *testing.T) {
 		c := p.GetPullRequestConfig()
 
 		assert.True(t, p.IsPullRequest())
+		assert.Equal(t, "refs/pull/42/head", p.GetReference())
 		assert.Equal(t, "feat/test-jenkins", c.Branch)
 		assert.Equal(t, "main", c.Base)
 		assert.Equal(t, "42", c.Key)

pkg/orchestrator/orchestrator.go

@@ -2,9 +2,10 @@ package orchestrator
 
 import (
 	"errors"
-	"github.com/SAP/jenkins-library/pkg/log"
 	"os"
 	"time"
+
+	"github.com/SAP/jenkins-library/pkg/log"
 )
 
 type Orchestrator int
@@ -22,6 +23,7 @@ type OrchestratorSpecificConfigProviding interface {
 	OrchestratorVersion() string
 	GetStageName() string
 	GetBranch() string
+	GetReference() string
 	GetBuildURL() string
 	GetBuildID() string
 	GetJobURL() string

pkg/orchestrator/unknownOrchestrator.go

@@ -1,8 +1,9 @@
 package orchestrator
 
 import (
-	"github.com/SAP/jenkins-library/pkg/log"
 	"time"
+
+	"github.com/SAP/jenkins-library/pkg/log"
 )
 
 type UnknownOrchestratorConfigProvider struct{}
@@ -72,6 +73,12 @@ func (u *UnknownOrchestratorConfigProvider) GetBranch() string {
 	return "n/a"
 }
 
+// GetReference returns n/a for the unknownOrchestrator
+func (u *UnknownOrchestratorConfigProvider) GetReference() string {
+	log.Entry().Warning("Unknown orchestrator - returning default values.")
+	return "n/a"
+}
+
 // GetBuildURL returns n/a for the unknownOrchestrator
 func (u *UnknownOrchestratorConfigProvider) GetBuildURL() string {
 	log.Entry().Warning("Unknown orchestrator - returning default values.")

resources/metadata/codeqlExecuteScan.yaml

@@ -0,0 +1,125 @@
+metadata:
+  name: codeqlExecuteScan
+  description: This step executes a codeql scan on the specified project to perform static code analysis and check the source code for security flaws.
+  longDescription: |-
+    This step executes a codeql scan on the specified project to perform static code analysis and check the source code for security flaws.
+
+    The codeql step triggers a scan locally on your orchestrator (e.g. Jenkins) within a docker container so finally you have to supply a docker image with codeql
+    and Java plus Maven.
+
+spec:
+  inputs:
+    secrets:
+      - name: githubTokenCredentialsId
+        description: Jenkins 'Secret text' credentials ID containing token to authenticate to GitHub.
+        type: jenkins
+    params:
+      - name: githubToken
+        description: "GitHub personal access token as per
+          https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line"
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: string
+        secret: true
+        aliases:
+          - name: access_token
+        resourceRef:
+          - name: githubTokenCredentialsId
+            type: secret
+          - type: vaultSecret
+            default: github
+            name: githubVaultSecretName
+      - name: buildTool
+        type: string
+        description: Defines the build tool which is used for building the project.
+        longDescription: |-
+          Based on the build tool the step will try to auto build the project. The step will try to auto select
+          the language and the build command. You can override the language and the build command by specifiying it seperatly.
+        mandatory: true
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        possibleValues:
+          - custom
+          - maven
+          - golang
+          - npm
+          - pip
+          - yarn
+        default: "maven"
+      - name: buildCommand
+        type: string
+        description: "Command to build the project"
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+      - name: language
+        type: string
+        description: "The programming language used to analyze."
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+      - name: modulePath
+        type: string
+        description: "Allows providing the path for the module to scan"
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        default: "./"
+      - name: querySuite
+        type: string
+        description: "The name of a CodeQL query suite. If omitted, the default query suite for the language of the database being analyzed will be used."
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+      - name: uploadResults
+        type: bool
+        description: "Allows you to upload codeql SARIF results to your github project. You will need to set githubToken for this."
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        default: false
+      - name: analyzedRef
+        type: string
+        description: "Name of the ref that was analyzed."
+        longDescription: |-
+          If this ref is a pull request merge commit, then use refs/pulls/1234/merge or refs/pulls/1234/head (depending on whether or not this commit corresponds to the HEAD or MERGE commit of the PR).
+          Otherwise, this should be a branch: refs/heads/branch-name. If omitted, the CLI will attempt to automatically populate this from the current branch of the checkout path, if this exists.
+        resourceRef:
+          - name: commonPipelineEnvironment
+            param: git/ref
+      - name: repository
+        aliases:
+          - name: githubRepo
+        description: "URL of the GitHub instance"
+        resourceRef:
+          - name: commonPipelineEnvironment
+            param: git/httpsUrl
+        type: string
+      - name: commitId
+        description: "SHA of commit that was analyzed."
+        resourceRef:
+          - name: commonPipelineEnvironment
+            param: git/commitId
+        type: string
+  containers:
+    - image: ""
+  outputs:
+    resources:
+      - name: reports
+        type: reports
+        params:
+          - filePattern: "**/*.csv"
+            type: codeql
+          - filePattern: "**/*.sarif"
+            type: codeql

test/groovy/CommonStepsTest.groovy

@@ -173,6 +173,7 @@ public class CommonStepsTest extends BasePiperTest{
         'gctsExecuteABAPQualityChecks', //implementing new golang pattern without fields
         'gctsExecuteABAPUnitTests', //implementing new golang pattern without fields
         'gctsCloneRepository', //implementing new golang pattern without fields
+        'codeqlExecuteScan', //implementing new golang pattern without fields
         'fortifyExecuteScan', //implementing new golang pattern without fields
         'gctsDeploy', //implementing new golang pattern without fields
         'containerSaveImage', //implementing new golang pattern without fields

test/groovy/SetupCommonPipelineEnvironmentTest.groovy

@@ -307,6 +307,30 @@ class SetupCommonPipelineEnvironmentTest extends BasePiperTest {
         assertThat(nullScript.commonPipelineEnvironment.gitCommitId, is('dummy_git_commit_id'))
     }
 
+    @Test
+    void "Set scmInfo parameter sets git reference for branch"() {
+        helper.registerAllowedMethod("fileExists", [String], { String path ->
+            return path.endsWith('.pipeline/config.yml')
+        })
+
+        def dummyScmInfo = [GIT_BRANCH: 'origin/testbranch']
+
+        stepRule.step.setupCommonPipelineEnvironment(script: nullScript, scmInfo: dummyScmInfo)
+        assertThat(nullScript.commonPipelineEnvironment.gitRef, is('refs/heads/testbranch'))
+    }
+
+    @Test
+    void "Set scmInfo parameter sets git reference for pull request"() {
+        helper.registerAllowedMethod("fileExists", [String], { String path ->
+            return path.endsWith('.pipeline/config.yml')
+        })
+
+        def dummyScmInfo = [GIT_BRANCH: 'PR-42']
+
+        stepRule.step.setupCommonPipelineEnvironment(script: nullScript, scmInfo: dummyScmInfo)
+        assertThat(nullScript.commonPipelineEnvironment.gitRef, is('refs/pull/42/head'))
+    }
+
     @Test
     void "No scmInfo passed as parameter yields empty git info"() {
         helper.registerAllowedMethod("fileExists", [String], { String path ->
@@ -319,6 +343,7 @@ class SetupCommonPipelineEnvironmentTest extends BasePiperTest {
         assertNull(nullScript.commonPipelineEnvironment.getGitHttpsUrl())
         assertNull(nullScript.commonPipelineEnvironment.getGithubOrg())
         assertNull(nullScript.commonPipelineEnvironment.getGithubRepo())
+        assertNull(nullScript.commonPipelineEnvironment.getGitRef())
     }
 
     @Test

vars/codeqlExecuteScan.groovy

@@ -0,0 +1,9 @@
+import groovy.transform.Field
+
+@Field String STEP_NAME = getClass().getName()
+@Field String METADATA_FILE = 'metadata/codeqlExecuteScan.yaml'
+
+void call(Map parameters = [:]) {
+    List credentials = [[type: 'token', id: 'githubTokenCredentialsId', env: ['PIPER_githubToken']]]
+    piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials)
+}

vars/commonPipelineEnvironment.groovy

@@ -30,6 +30,7 @@ class commonPipelineEnvironment implements Serializable {
     String gitSshUrl
     String gitHttpsUrl
     String gitBranch
+    String gitRef
 
     String xsDeploymentId
 
@@ -88,6 +89,7 @@ class commonPipelineEnvironment implements Serializable {
         gitSshUrl = null
         gitHttpsUrl = null
         gitBranch = null
+        gitRef = null
 
         githubOrg = null
         githubRepo = null
@@ -199,6 +201,7 @@ class commonPipelineEnvironment implements Serializable {
         [filename: '.pipeline/commonPipelineEnvironment/git/branch', property: 'gitBranch'],
         [filename: '.pipeline/commonPipelineEnvironment/git/commitId', property: 'gitCommitId'],
         [filename: '.pipeline/commonPipelineEnvironment/git/httpsUrl', property: 'gitHttpsUrl'],
+        [filename: '.pipeline/commonPipelineEnvironment/git/ref', property: 'gitRef'],
         [filename: '.pipeline/commonPipelineEnvironment/git/commitMessage', property: 'gitCommitMessage'],
         [filename: '.pipeline/commonPipelineEnvironment/mtarFilePath', property: 'mtarFilePath'],
         [filename: '.pipeline/commonPipelineEnvironment/abap/addonDescriptor', property: 'abapAddonDescriptor'],

vars/setupCommonPipelineEnvironment.groovy

@@ -118,6 +118,7 @@ void call(Map parameters = [:]) {
         if (scmInfo) {
             setGitUrlsOnCommonPipelineEnvironment(script, scmInfo.GIT_URL)
             script.commonPipelineEnvironment.setGitCommitId(scmInfo.GIT_COMMIT)
+            setGitRefOnCommonPipelineEnvironment(script, scmInfo.GIT_BRANCH)
         }
     }
 }
@@ -259,3 +260,20 @@ private void setGitUrlsOnCommonPipelineEnvironment(script, String gitUrl) {
     script.commonPipelineEnvironment.setGithubOrg(gitFolder)
     script.commonPipelineEnvironment.setGithubRepo(gitRepo)
 }
+
+private void setGitRefOnCommonPipelineEnvironment(script, String gitBranch) {
+    if(!gitBranch){
+        return
+    }
+
+    if(gitBranch.contains("/")){
+        gitBranch = gitBranch.split("/")[1]
+    }
+
+    //TODO: refs for merge pull requests
+    if (gitBranch.contains("PR")) {
+		script.commonPipelineEnvironment.setGitRef("refs/pull/" + gitBranch.split("-")[1] + "/head")
+	} else {
+		script.commonPipelineEnvironment.setGitRef("refs/heads/" + gitBranch)
+	}
+}