Add credentialdiggerScan step (#4141)

* Add credentialdiggerScan metadata

* Integrate new step into piper process

* Add credentialdiggerScan implementation and tests

* Remove duplicated code

* Add doc file for credentialdiggerScan step

* Regenerate metadata

* Fix return type in tests

* Add credentialdiggerScan to CommonStepsTest

* Fix typo

* Improve code style

* Add support for custom rules file in stash

* Regenerate metadata for credentialdiggerScan

---------

Co-authored-by: Sven Merk <33895725+nevskrem@users.noreply.github.com>
Co-authored-by: Anil Keshav <anil.keshav@sap.com>

cmd/credentialdiggerScan.go

@@ -0,0 +1,249 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/SAP/jenkins-library/pkg/command"
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/orchestrator"
+	"github.com/SAP/jenkins-library/pkg/piperutils"
+	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/pkg/errors"
+)
+
+const piperDbName string = "piper_step_db.db"
+const piperReportName string = "findings.csv"
+
+type credentialdiggerUtils interface {
+	command.ExecRunner
+	piperutils.FileUtils
+}
+
+type credentialdiggerUtilsBundle struct {
+	*command.Command
+	*piperutils.Files
+}
+
+func newCDUtils() credentialdiggerUtils {
+	utils := credentialdiggerUtilsBundle{
+		Command: &command.Command{},
+		Files:   &piperutils.Files{},
+	}
+	// Reroute command output to logging framework
+	utils.Stdout(log.Writer())
+	utils.Stderr(log.Writer())
+	return &utils
+}
+
+func credentialdiggerScan(config credentialdiggerScanOptions, telemetryData *telemetry.CustomData) error {
+	utils := newCDUtils()
+	// 0: Get attributes from orchestrator
+	provider, prov_err := orchestrator.NewOrchestratorSpecificConfigProvider()
+	if prov_err != nil {
+		log.Entry().WithError(prov_err).Error(
+			"credentialdiggerScan: unable to load orchestrator specific configuration.")
+	}
+	if config.Repository == "" {
+		// Get current repository from orchestrator
+		repoUrlOrchestrator := provider.GetRepoURL()
+		if repoUrlOrchestrator == "n/a" {
+			// Jenkins configuration error
+			log.Entry().WithError(errors.New(
+				fmt.Sprintf("Unknown repository URL %s", repoUrlOrchestrator))).Error(
+				"Repository URL n/a. Please verify git plugin is installed.")
+		}
+		config.Repository = repoUrlOrchestrator
+		log.Entry().Debug("Use current repository: ", repoUrlOrchestrator)
+	}
+	if provider.IsPullRequest() {
+		// set the pr number
+		config.PrNumber, _ = strconv.Atoi(provider.GetPullRequestConfig().Key)
+		log.Entry().Debug("Scan the current pull request: number ", config.PrNumber)
+	}
+
+	// 1: Add rules
+	log.Entry().Info("Load rules")
+	err := credentialdiggerAddRules(&config, telemetryData, utils)
+	if err != nil {
+		log.Entry().Error("credentialdiggerScan: Failed running credentialdigger add_rules")
+		return err
+	}
+	log.Entry().Info("Rules added")
+
+	// 2: Scan the repository
+	// Choose between scan-pr, scan-snapshot, and full-scan (with this priority
+	// order)
+	switch {
+	case config.PrNumber != 0: // int type is not nillable in golang
+		log.Entry().Debug("Scan PR")
+		// if a PrNumber is declared, run scan_pr
+		err = credentialdiggerScanPR(&config, telemetryData, utils) // scan PR with CD
+	case config.Snapshot != "":
+		log.Entry().Debug("Scan snapshot")
+		// if a Snapshot is declared, run scan_snapshot
+		err = credentialdiggerScanSnapshot(&config, telemetryData, utils) // scan Snapshot with CD
+	default:
+		// The default case is the normal full scan
+		log.Entry().Debug("Full scan repo")
+		err = credentialdiggerFullScan(&config, telemetryData, utils) // full scan with CD
+	}
+	// err is an error exit number when there are findings
+	if err == nil {
+		log.Entry().Info("No discoveries found in this repo")
+		// If there are no findings, there is no need to export an empty report
+		return nil
+	}
+
+	// 3: Get discoveries
+	err = credentialdiggerGetDiscoveries(&config, telemetryData, utils)
+	if err != nil {
+		// The exit number is the number of discoveries
+		// Therefore, this error is not relevant, if raised
+		log.Entry().Warn("There are findings to review")
+	}
+
+	// 4: Export report in workspace
+	reports := []piperutils.Path{}
+	reports = append(reports, piperutils.Path{Target: fmt.Sprintf("%v", piperReportName)})
+	piperutils.PersistReportsAndLinks("credentialdiggerScan", "./", utils, reports, nil)
+
+	return nil
+}
+
+func executeCredentialDiggerProcess(utils credentialdiggerUtils, args []string) error {
+	return utils.RunExecutable("credentialdigger", args...)
+}
+
+// hasConfigurationFile checks if the given file exists
+func hasRulesFile(file string, utils credentialdiggerUtils) bool {
+	exists, err := utils.FileExists(file)
+	if err != nil {
+		log.Entry().WithError(err).Error()
+	}
+	return exists
+}
+
+func credentialdiggerAddRules(config *credentialdiggerScanOptions, telemetryData *telemetry.CustomData, service credentialdiggerUtils) error {
+	// Credentialdigger home can be changed with local forks (e.g., for local piper runs)
+	cdHome := "/credential-digger-ui" // cdHome path as in docker container
+	if cdh := os.Getenv("CREDENTIALDIGGER_HOME"); cdh != "" {
+		cdHome = cdh
+	}
+	log.Entry().Debug("Use credentialdigger home ", cdHome)
+	// Set the rule file to the standard ruleset shipped within credential
+	// digger container
+	ruleFile := filepath.Join(cdHome, "backend", "rules.yml")
+
+	if config.RulesDownloadURL != "" {
+		// Download custom rule file from this URL
+		log.Entry().Debugf("Download custom ruleset from %v", config.RulesDownloadURL)
+		dlClient := piperhttp.Client{}
+		ruleFile := filepath.Join(cdHome, "backend", "custom-rules.yml")
+		dlClient.DownloadFile(config.RulesDownloadURL, ruleFile, nil, nil)
+		log.Entry().Info("Download and use remote rules")
+	} else {
+		log.Entry().Debug("Use a local ruleset")
+		// Use rules defined in stashed file
+		if hasRulesFile(config.RulesFile, service) {
+			log.Entry().WithField("file", config.RulesFile).Info("Use stashed rules file from repository")
+			ruleFile = config.RulesFile
+		} else {
+			log.Entry().Info("Use standard pre-defined rules")
+		}
+
+	}
+	cmd_list := []string{"add_rules", "--sqlite", piperDbName, ruleFile}
+	return executeCredentialDiggerProcess(service, cmd_list)
+}
+
+func credentialdiggerGetDiscoveries(config *credentialdiggerScanOptions, telemetryData *telemetry.CustomData, service credentialdiggerUtils) error {
+	log.Entry().Info("Get discoveries")
+	cmd_list := []string{"get_discoveries", config.Repository, "--sqlite", piperDbName,
+		"--save", piperReportName}
+	// Export all the discoveries or export only new ones
+	if !config.ExportAll {
+		cmd_list = append(cmd_list, "--state", "new")
+	}
+	err := executeCredentialDiggerProcess(service, cmd_list)
+	if err != nil {
+		log.Entry().Error("credentialdiggerScan: Failed running credentialdigger get_discoveries")
+		log.Entry().Error(err)
+		return err
+	}
+	log.Entry().Info("Scan complete")
+	return nil
+}
+
+func credentialdiggerBuildCommonArgs(config *credentialdiggerScanOptions) []string {
+	/*Some arguments are the same for all the scan flavors. Build them here
+	* not to duplicate code.*/
+	scan_args := []string{}
+	// Repository url and sqlite db (always mandatory)
+	scan_args = append(scan_args, config.Repository, "--sqlite", piperDbName)
+	//git token is not mandatory for base credential digger tool, but in
+	//piper it is
+	scan_args = append(scan_args, "--git_token", config.Token)
+	//debug
+	if config.Debug {
+		log.Entry().Debug("Run the scan in debug mode")
+		scan_args = append(scan_args, "--debug")
+	}
+	//models
+	if len(config.Models) > 0 {
+		log.Entry().Debugf("Enable models %v", config.Models)
+		scan_args = append(scan_args, "--models")
+		scan_args = append(scan_args, config.Models...)
+	}
+
+	return scan_args
+}
+
+func credentialdiggerScanSnapshot(config *credentialdiggerScanOptions, telemetryData *telemetry.CustomData, service credentialdiggerUtils) error {
+	log.Entry().Infof("Scan Snapshot %v from repo %v", config.Snapshot, config.Repository)
+	cmd_list := []string{"scan_snapshot",
+		"--snapshot", config.Snapshot}
+	cmd_list = append(cmd_list, credentialdiggerBuildCommonArgs(config)...)
+	leaks := executeCredentialDiggerProcess(service, cmd_list)
+	if leaks != nil {
+		log.Entry().Warn("The scan found potential leaks in this Snapshot")
+		return leaks
+	} else {
+		log.Entry().Info("No leaks found")
+		return nil
+	}
+}
+
+func credentialdiggerScanPR(config *credentialdiggerScanOptions, telemetryData *telemetry.CustomData, service credentialdiggerUtils) error {
+	log.Entry().Infof("Scan PR %v from repo %v", config.PrNumber, config.Repository)
+	cmd_list := []string{"scan_pr",
+		"--pr", strconv.Itoa(config.PrNumber),
+		"--api_endpoint", config.APIURL}
+	cmd_list = append(cmd_list, credentialdiggerBuildCommonArgs(config)...)
+	leaks := executeCredentialDiggerProcess(service, cmd_list)
+	if leaks != nil {
+		log.Entry().Warn("The scan found potential leaks in this PR")
+		return leaks
+	} else {
+		log.Entry().Info("No leaks found")
+		return nil
+	}
+}
+
+func credentialdiggerFullScan(config *credentialdiggerScanOptions, telemetryData *telemetry.CustomData, service credentialdiggerUtils) error {
+	log.Entry().Infof("Full scan of repository %v", config.Repository)
+	cmd_list := []string{"scan"}
+	cmd_list = append(cmd_list, credentialdiggerBuildCommonArgs(config)...)
+	leaks := executeCredentialDiggerProcess(service, cmd_list)
+	if leaks != nil {
+		log.Entry().Warn("The scan found potential leaks")
+		log.Entry().Warnf("%v potential leaks found", leaks)
+		return leaks
+	} else {
+		log.Entry().Info("No leaks found")
+		return nil
+	}
+}

cmd/credentialdiggerScan_generated.go

@@ -0,0 +1,277 @@
+// Code generated by piper's step-generator. DO NOT EDIT.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/SAP/jenkins-library/pkg/config"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/splunk"
+	"github.com/SAP/jenkins-library/pkg/telemetry"
+	"github.com/SAP/jenkins-library/pkg/validation"
+	"github.com/spf13/cobra"
+)
+
+type credentialdiggerScanOptions struct {
+	Repository       string   `json:"repository,omitempty"`
+	Snapshot         string   `json:"snapshot,omitempty"`
+	PrNumber         int      `json:"prNumber,omitempty"`
+	ExportAll        bool     `json:"exportAll,omitempty"`
+	APIURL           string   `json:"apiUrl,omitempty"`
+	Debug            bool     `json:"debug,omitempty"`
+	RulesDownloadURL string   `json:"rulesDownloadUrl,omitempty"`
+	Models           []string `json:"models,omitempty"`
+	Token            string   `json:"token,omitempty"`
+	RulesFile        string   `json:"rulesFile,omitempty"`
+}
+
+// CredentialdiggerScanCommand Scan a repository on GitHub with Credential Digger
+func CredentialdiggerScanCommand() *cobra.Command {
+	const STEP_NAME = "credentialdiggerScan"
+
+	metadata := credentialdiggerScanMetadata()
+	var stepConfig credentialdiggerScanOptions
+	var startTime time.Time
+	var logCollector *log.CollectorHook
+	var splunkClient *splunk.Splunk
+	telemetryClient := &telemetry.Telemetry{}
+
+	var createCredentialdiggerScanCmd = &cobra.Command{
+		Use:   STEP_NAME,
+		Short: "Scan a repository on GitHub with Credential Digger",
+		Long: `This step allows you to scan a repository on Github using Credential Digger.
+
+It can for example be used for DevSecOps scenarios to verify the source code does not contain hard-coded credentials before being merged or released for production.
+It supports several scan flavors, i.e., full scans of a repo, scan of a snapshot, or scan of a pull request.`,
+		PreRunE: func(cmd *cobra.Command, _ []string) error {
+			startTime = time.Now()
+			log.SetStepName(STEP_NAME)
+			log.SetVerbose(GeneralConfig.Verbose)
+
+			GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens)
+
+			path, _ := os.Getwd()
+			fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path}
+			log.RegisterHook(fatalHook)
+
+			err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile)
+			if err != nil {
+				log.SetErrorCategory(log.ErrorConfiguration)
+				return err
+			}
+			log.RegisterSecret(stepConfig.Token)
+
+			if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 {
+				sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID)
+				log.RegisterHook(&sentryHook)
+			}
+
+			if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+				splunkClient = &splunk.Splunk{}
+				logCollector = &log.CollectorHook{CorrelationID: GeneralConfig.CorrelationID}
+				log.RegisterHook(logCollector)
+			}
+
+			if err = log.RegisterANSHookIfConfigured(GeneralConfig.CorrelationID); err != nil {
+				log.Entry().WithError(err).Warn("failed to set up SAP Alert Notification Service log hook")
+			}
+
+			validation, err := validation.New(validation.WithJSONNamesForStructFields(), validation.WithPredefinedErrorMessages())
+			if err != nil {
+				return err
+			}
+			if err = validation.ValidateStruct(stepConfig); err != nil {
+				log.SetErrorCategory(log.ErrorConfiguration)
+				return err
+			}
+
+			return nil
+		},
+		Run: func(_ *cobra.Command, _ []string) {
+			stepTelemetryData := telemetry.CustomData{}
+			stepTelemetryData.ErrorCode = "1"
+			handler := func() {
+				config.RemoveVaultSecretFiles()
+				stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds())
+				stepTelemetryData.ErrorCategory = log.GetErrorCategory().String()
+				stepTelemetryData.PiperCommitHash = GitCommit
+				telemetryClient.SetData(&stepTelemetryData)
+				telemetryClient.Send()
+				if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+					splunkClient.Send(telemetryClient.GetData(), logCollector)
+				}
+			}
+			log.DeferExitHandler(handler)
+			defer handler()
+			telemetryClient.Initialize(GeneralConfig.NoTelemetry, STEP_NAME)
+			if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
+				splunkClient.Initialize(GeneralConfig.CorrelationID,
+					GeneralConfig.HookConfig.SplunkConfig.Dsn,
+					GeneralConfig.HookConfig.SplunkConfig.Token,
+					GeneralConfig.HookConfig.SplunkConfig.Index,
+					GeneralConfig.HookConfig.SplunkConfig.SendLogs)
+			}
+			credentialdiggerScan(stepConfig, &stepTelemetryData)
+			stepTelemetryData.ErrorCode = "0"
+			log.Entry().Info("SUCCESS")
+		},
+	}
+
+	addCredentialdiggerScanFlags(createCredentialdiggerScanCmd, &stepConfig)
+	return createCredentialdiggerScanCmd
+}
+
+func addCredentialdiggerScanFlags(cmd *cobra.Command, stepConfig *credentialdiggerScanOptions) {
+	cmd.Flags().StringVar(&stepConfig.Repository, "repository", os.Getenv("PIPER_repository"), "URL of the GitHub repository (was name, but we need the url). In case it's missing, use the URL of the current repository.")
+	cmd.Flags().StringVar(&stepConfig.Snapshot, "snapshot", os.Getenv("PIPER_snapshot"), "If set, scan the snapshot of the repository at this commit_id/branch.")
+	cmd.Flags().IntVar(&stepConfig.PrNumber, "prNumber", 0, "If set, scan the pull request open with this number.")
+	cmd.Flags().BoolVar(&stepConfig.ExportAll, "exportAll", false, "Export all the findings, i.e., including non-leaks.")
+	cmd.Flags().StringVar(&stepConfig.APIURL, "apiUrl", `https://api.github.com`, "Set the GitHub API url. Needed for scanning a pull request.")
+	cmd.Flags().BoolVar(&stepConfig.Debug, "debug", false, "Execute the scans in debug mode (i.e., print logs).")
+	cmd.Flags().StringVar(&stepConfig.RulesDownloadURL, "rulesDownloadUrl", os.Getenv("PIPER_rulesDownloadUrl"), "URL where to download custom rules. The file published at this URL must be formatted as the default ruleset https://raw.githubusercontent.com/SAP/credential-digger/main/ui/backend/rules.yml")
+	cmd.Flags().StringSliceVar(&stepConfig.Models, "models", []string{}, "Machine learning models to automatically verify the findings.")
+	cmd.Flags().StringVar(&stepConfig.Token, "token", os.Getenv("PIPER_token"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line")
+	cmd.Flags().StringVar(&stepConfig.RulesFile, "rulesFile", `inputs/rules.yml`, "Name of the rules file used locally within the step. If a remote files for rules is declared as `rulesDownloadUrl`, the stashed file is ignored. If you change the file's name make sure your stashing configuration also reflects this.")
+
+	cmd.MarkFlagRequired("apiUrl")
+	cmd.MarkFlagRequired("token")
+}
+
+// retrieve step metadata
+func credentialdiggerScanMetadata() config.StepData {
+	var theMetaData = config.StepData{
+		Metadata: config.StepMetadata{
+			Name:        "credentialdiggerScan",
+			Aliases:     []config.Alias{},
+			Description: "Scan a repository on GitHub with Credential Digger",
+		},
+		Spec: config.StepSpec{
+			Inputs: config.StepInputs{
+				Secrets: []config.StepSecrets{
+					{Name: "githubTokenCredentialsId", Description: "Jenkins 'Secret text' credentials ID containing token to authenticate to GitHub.", Type: "jenkins"},
+				},
+				Parameters: []config.StepParameters{
+					{
+						Name:        "repository",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{{Name: "githubRepo"}},
+						Default:     os.Getenv("PIPER_repository"),
+					},
+					{
+						Name:        "snapshot",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_snapshot"),
+					},
+					{
+						Name:        "prNumber",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "int",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     0,
+					},
+					{
+						Name:        "exportAll",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "bool",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     false,
+					},
+					{
+						Name:        "apiUrl",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   true,
+						Aliases:     []config.Alias{{Name: "githubApiUrl"}},
+						Default:     `https://api.github.com`,
+					},
+					{
+						Name:        "debug",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:        "bool",
+						Mandatory:   false,
+						Aliases:     []config.Alias{{Name: "verbose"}},
+						Default:     false,
+					},
+					{
+						Name:        "rulesDownloadUrl",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     os.Getenv("PIPER_rulesDownloadUrl"),
+					},
+					{
+						Name:        "models",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "[]string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     []string{},
+					},
+					{
+						Name: "token",
+						ResourceRef: []config.ResourceReference{
+							{
+								Name: "githubTokenCredentialsId",
+								Type: "secret",
+							},
+
+							{
+								Name:    "githubVaultSecretName",
+								Type:    "vaultSecret",
+								Default: "github",
+							},
+						},
+						Scope:     []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
+						Type:      "string",
+						Mandatory: true,
+						Aliases:   []config.Alias{{Name: "githubToken"}, {Name: "access_token"}},
+						Default:   os.Getenv("PIPER_token"),
+					},
+					{
+						Name:        "rulesFile",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "string",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+						Default:     `inputs/rules.yml`,
+					},
+				},
+			},
+			Containers: []config.Container{
+				{Image: "credentialdigger.int.repositories.cloud.sap/credential_digger:4.9.2"},
+			},
+			Outputs: config.StepOutputs{
+				Resources: []config.StepResources{
+					{
+						Name: "report",
+						Type: "report",
+						Parameters: []map[string]interface{}{
+							{"filePattern": "**/report*.csv", "type": "credentialdigger-report"},
+						},
+					},
+				},
+			},
+		},
+	}
+	return theMetaData
+}

cmd/credentialdiggerScan_generated_test.go

@@ -0,0 +1,17 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCredentialdiggerScanCommand(t *testing.T) {
+	t.Parallel()
+
+	testCmd := CredentialdiggerScanCommand()
+
+	// only high level testing performed - details are tested in step generation procedure
+	assert.Equal(t, "credentialdiggerScan", testCmd.Use, "command name incorrect")
+
+}

cmd/credentialdiggerScan_test.go

@@ -0,0 +1,124 @@
+package cmd
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/SAP/jenkins-library/pkg/mock"
+	"github.com/stretchr/testify/assert"
+)
+
+type credentialdiggerScanMockUtils struct {
+	*mock.ExecMockRunner
+	*mock.FilesMock
+	noerr bool
+}
+
+func newCDTestsUtils() credentialdiggerScanMockUtils {
+	utils := credentialdiggerScanMockUtils{
+		ExecMockRunner: &mock.ExecMockRunner{},
+		FilesMock:      &mock.FilesMock{},
+		noerr:          true, // flag for return value of MockRunner
+	}
+	return utils
+}
+func (c credentialdiggerScanMockUtils) RunExecutable(executable string, params ...string) error {
+	if c.noerr {
+		return nil
+	} else {
+		return errors.New("Some custom error")
+	}
+}
+
+func TestCredentialdiggerFullScan(t *testing.T) {
+	t.Run("Valid full scan without discoveries", func(t *testing.T) {
+		config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken"}
+		utils := newCDTestsUtils()
+		assert.Equal(t, nil, credentialdiggerFullScan(&config, nil, utils))
+
+	})
+	t.Run("Full scan with discoveries or wrong arguments", func(t *testing.T) {
+		config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken"}
+		utils := newCDTestsUtils()
+		utils.noerr = false
+		assert.EqualError(t, credentialdiggerFullScan(&config, nil, utils), "Some custom error")
+	})
+}
+
+func TestCredentialdiggerScanSnapshot(t *testing.T) {
+	t.Run("Valid scan snapshot without discoveries", func(t *testing.T) {
+		config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken", Snapshot: "main"}
+		utils := newCDTestsUtils()
+		assert.Equal(t, nil, credentialdiggerScanSnapshot(&config, nil, utils))
+	})
+	t.Run("Scan snapshot with discoveries or wrong arguments", func(t *testing.T) {
+		config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken", Snapshot: "main"}
+		utils := newCDTestsUtils()
+		utils.noerr = false
+		assert.EqualError(t, credentialdiggerScanSnapshot(&config, nil, utils), "Some custom error")
+	})
+}
+
+func TestCredentialdiggerScanPR(t *testing.T) {
+	t.Run("Valid scan pull request without discoveries", func(t *testing.T) {
+		config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken", PrNumber: 1}
+		utils := newCDTestsUtils()
+		assert.Equal(t, nil, credentialdiggerScanPR(&config, nil, utils))
+	})
+	t.Run("Scan pull request with discoveries or wrong arguments", func(t *testing.T) {
+		config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken", PrNumber: 1}
+		utils := newCDTestsUtils()
+		utils.noerr = false
+		assert.EqualError(t, credentialdiggerScanPR(&config, nil, utils), "Some custom error")
+	})
+}
+
+func TestCredentialdiggerAddRules(t *testing.T) {
+	t.Run("Valid standard or remote rules", func(t *testing.T) {
+		config := credentialdiggerScanOptions{}
+		utils := newCDTestsUtils()
+		assert.Equal(t, nil, credentialdiggerAddRules(&config, nil, utils))
+	})
+	t.Run("Broken add rules", func(t *testing.T) {
+		config := credentialdiggerScanOptions{}
+		utils := newCDTestsUtils()
+		utils.noerr = false
+		assert.EqualError(t, credentialdiggerAddRules(&config, nil, utils), "Some custom error")
+	})
+	/*
+		// In case we want to test the error raised by piperhttp
+		t.Run("Invalid external rules link", func(t *testing.T) {
+			rulesExt := "https://broken-link.com/fakerules"
+			config := credentialdiggerScanOptions{RulesDownloadURL: rulesExt}
+			utils := newCDTestsUtils()
+			assert.Equal(t, nil, credentialdiggerAddRules(&config, nil, utils))
+		})
+	*/
+}
+
+func TestCredentialdiggerGetDiscoveries(t *testing.T) {
+	t.Run("Empty discoveries", func(t *testing.T) {
+		config := credentialdiggerScanOptions{Repository: "testRepo"}
+		utils := newCDTestsUtils()
+		assert.Equal(t, nil, credentialdiggerGetDiscoveries(&config, nil, utils))
+	})
+	t.Run("Get discoveries non-empty", func(t *testing.T) {
+		config := credentialdiggerScanOptions{Repository: "testRepo"}
+		utils := newCDTestsUtils()
+		utils.noerr = false
+		assert.EqualError(t, credentialdiggerGetDiscoveries(&config, nil, utils), "Some custom error")
+	})
+}
+
+func TestCredentialdiggerBuildCommonArgs(t *testing.T) {
+	t.Run("Valid build common args", func(t *testing.T) {
+		arguments := []string{"repoURL", "--sqlite", "piper_step_db.db", "--git_token", "validToken",
+			"--debug", "--models", "model1", "model2"}
+		config := credentialdiggerScanOptions{Repository: "repoURL", Token: "validToken", Snapshot: "main",
+			Debug: true, PrNumber: 1,
+			Models: []string{"model1", "model2"},
+		}
+		assert.Equal(t, arguments, credentialdiggerBuildCommonArgs(&config))
+	})
+
+}

cmd/metadata_generated.go

@@ -49,6 +49,7 @@ func GetAllStepMetadata() map[string]config.StepData {
 		"codeqlExecuteScan":                         codeqlExecuteScanMetadata(),
 		"containerExecuteStructureTests":            containerExecuteStructureTestsMetadata(),
 		"containerSaveImage":                        containerSaveImageMetadata(),
+		"credentialdiggerScan":                      credentialdiggerScanMetadata(),
 		"detectExecuteScan":                         detectExecuteScanMetadata(),
 		"fortifyExecuteScan":                        fortifyExecuteScanMetadata(),
 		"gaugeExecuteTests":                         gaugeExecuteTestsMetadata(),

cmd/piper.go

@@ -113,6 +113,7 @@ func Execute() {
 	rootCmd.AddCommand(CheckmarxExecuteScanCommand())
 	rootCmd.AddCommand(FortifyExecuteScanCommand())
 	rootCmd.AddCommand(CodeqlExecuteScanCommand())
+	rootCmd.AddCommand(CredentialdiggerScanCommand())
 	rootCmd.AddCommand(MtaBuildCommand())
 	rootCmd.AddCommand(ProtecodeExecuteScanCommand())
 	rootCmd.AddCommand(MavenExecuteCommand())

documentation/docs/steps/credentialdiggerScan.md

@@ -0,0 +1,7 @@
+# ${docGenStepName}
+
+## ${docGenDescription}
+
+## ${docGenParameters}
+
+## ${docGenConfiguration}

documentation/mkdocs.yml

@@ -94,6 +94,7 @@ nav:
         - commonPipelineEnvironment: steps/commonPipelineEnvironment.md
         - containerExecuteStructureTests: steps/containerExecuteStructureTests.md
         - containerPushToRegistry: steps/containerPushToRegistry.md
+        - credentialdiggerScan: steps/credentialdiggerScan.md
         - debugReportArchive: steps/debugReportArchive.md
         - detectExecuteScan: steps/detectExecuteScan.md
         - dockerExecute: steps/dockerExecute.md

resources/metadata/credentialdiggerScan.yaml

@@ -0,0 +1,124 @@
+metadata:
+  name: credentialdiggerScan
+  description: Scan a repository on GitHub with Credential Digger
+  longDescription: |
+    This step allows you to scan a repository on Github using Credential Digger.
+
+    It can for example be used for DevSecOps scenarios to verify the source code does not contain hard-coded credentials before being merged or released for production.
+    It supports several scan flavors, i.e., full scans of a repo, scan of a snapshot, or scan of a pull request.
+spec:
+  inputs:
+    secrets:
+      - name: githubTokenCredentialsId
+        description: Jenkins 'Secret text' credentials ID containing token to authenticate to GitHub.
+        type: jenkins
+    params:
+      - name: repository
+        aliases:
+          - name: githubRepo
+        description: URL of the GitHub repository (was name, but we need the url). In case it's missing, use the URL of the current repository.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: string
+        mandatory: false
+      - name: snapshot
+        description: If set, scan the snapshot of the repository at this commit_id/branch.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: string
+        mandatory: false
+      - name: prNumber
+        description: If set, scan the pull request open with this number.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: int
+        mandatory: false
+      - name: exportAll
+        type: bool
+        description: Export all the findings, i.e., including non-leaks.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        default: false
+      - name: apiUrl
+        aliases:
+          - name: githubApiUrl
+        description: Set the GitHub API url. Needed for scanning a pull request.
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: string
+        default: https://api.github.com
+        mandatory: true
+      - name: debug
+        aliases:
+          - name: verbose
+        description: Execute the scans in debug mode (i.e., print logs).
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: bool
+        default: false
+      - name: rulesDownloadUrl
+        type: string
+        description: URL where to download custom rules. The file published at this URL must be formatted as the default ruleset https://raw.githubusercontent.com/SAP/credential-digger/main/ui/backend/rules.yml
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        mandatory: false
+      - name: models
+        description: Machine learning models to automatically verify the findings.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: "[]string"
+      - name: token
+        aliases:
+          - name: githubToken
+          - name: access_token
+        description: GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line
+        scope:
+          - GENERAL
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        type: string
+        mandatory: true
+        secret: true
+        resourceRef:
+          - name: githubTokenCredentialsId
+            type: secret
+          - type: vaultSecret
+            default: github
+            name: githubVaultSecretName
+      - name: rulesFile
+        type: string
+        description: Name of the rules file used locally within the step. If a remote files for rules is declared as `rulesDownloadUrl`, the stashed file is ignored. If you change the file's name make sure your stashing configuration also reflects this.
+        mandatory: false
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        default: inputs/rules.yml
+  outputs:
+    resources:
+      - name: report
+        type: report
+        params:
+          - filePattern: "**/report*.csv"
+            type: credentialdigger-report
+  containers:
+    - image: "credentialdigger.int.repositories.cloud.sap/credential_digger:4.9.2"

test/groovy/CommonStepsTest.groovy

@@ -174,6 +174,7 @@ public class CommonStepsTest extends BasePiperTest{
         'gctsExecuteABAPUnitTests', //implementing new golang pattern without fields
         'gctsCloneRepository', //implementing new golang pattern without fields
         'codeqlExecuteScan', //implementing new golang pattern without fields
+        'credentialdiggerScan', //implementing new golang pattern without fields
         'fortifyExecuteScan', //implementing new golang pattern without fields
         'gctsDeploy', //implementing new golang pattern without fields
         'containerSaveImage', //implementing new golang pattern without fields

vars/credentialdiggerScan.groovy

@@ -0,0 +1,11 @@
+import groovy.transform.Field
+
+@Field String STEP_NAME = getClass().getName()
+@Field String METADATA_FILE = 'metadata/credentialdiggerScan.yaml'
+
+void call(Map parameters = [:]) {
+    List credentials = [
+        [type: 'token', id: 'githubTokenCredentialsId', env: ['PIPER_token']]
+    ]
+    piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials)
+}