diff --git a/cmd/credentialdiggerScan.go b/cmd/credentialdiggerScan.go new file mode 100644 index 000000000..31e84b39b --- /dev/null +++ b/cmd/credentialdiggerScan.go @@ -0,0 +1,249 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + + "github.com/SAP/jenkins-library/pkg/command" + piperhttp "github.com/SAP/jenkins-library/pkg/http" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/orchestrator" + "github.com/SAP/jenkins-library/pkg/piperutils" + "github.com/SAP/jenkins-library/pkg/telemetry" + "github.com/pkg/errors" +) + +const piperDbName string = "piper_step_db.db" +const piperReportName string = "findings.csv" + +type credentialdiggerUtils interface { + command.ExecRunner + piperutils.FileUtils +} + +type credentialdiggerUtilsBundle struct { + *command.Command + *piperutils.Files +} + +func newCDUtils() credentialdiggerUtils { + utils := credentialdiggerUtilsBundle{ + Command: &command.Command{}, + Files: &piperutils.Files{}, + } + // Reroute command output to logging framework + utils.Stdout(log.Writer()) + utils.Stderr(log.Writer()) + return &utils +} + +func credentialdiggerScan(config credentialdiggerScanOptions, telemetryData *telemetry.CustomData) error { + utils := newCDUtils() + // 0: Get attributes from orchestrator + provider, prov_err := orchestrator.NewOrchestratorSpecificConfigProvider() + if prov_err != nil { + log.Entry().WithError(prov_err).Error( + "credentialdiggerScan: unable to load orchestrator specific configuration.") + } + if config.Repository == "" { + // Get current repository from orchestrator + repoUrlOrchestrator := provider.GetRepoURL() + if repoUrlOrchestrator == "n/a" { + // Jenkins configuration error + log.Entry().WithError(errors.New( + fmt.Sprintf("Unknown repository URL %s", repoUrlOrchestrator))).Error( + "Repository URL n/a. Please verify git plugin is installed.") + } + config.Repository = repoUrlOrchestrator + log.Entry().Debug("Use current repository: ", repoUrlOrchestrator) + } + if provider.IsPullRequest() { + // set the pr number + config.PrNumber, _ = strconv.Atoi(provider.GetPullRequestConfig().Key) + log.Entry().Debug("Scan the current pull request: number ", config.PrNumber) + } + + // 1: Add rules + log.Entry().Info("Load rules") + err := credentialdiggerAddRules(&config, telemetryData, utils) + if err != nil { + log.Entry().Error("credentialdiggerScan: Failed running credentialdigger add_rules") + return err + } + log.Entry().Info("Rules added") + + // 2: Scan the repository + // Choose between scan-pr, scan-snapshot, and full-scan (with this priority + // order) + switch { + case config.PrNumber != 0: // int type is not nillable in golang + log.Entry().Debug("Scan PR") + // if a PrNumber is declared, run scan_pr + err = credentialdiggerScanPR(&config, telemetryData, utils) // scan PR with CD + case config.Snapshot != "": + log.Entry().Debug("Scan snapshot") + // if a Snapshot is declared, run scan_snapshot + err = credentialdiggerScanSnapshot(&config, telemetryData, utils) // scan Snapshot with CD + default: + // The default case is the normal full scan + log.Entry().Debug("Full scan repo") + err = credentialdiggerFullScan(&config, telemetryData, utils) // full scan with CD + } + // err is an error exit number when there are findings + if err == nil { + log.Entry().Info("No discoveries found in this repo") + // If there are no findings, there is no need to export an empty report + return nil + } + + // 3: Get discoveries + err = credentialdiggerGetDiscoveries(&config, telemetryData, utils) + if err != nil { + // The exit number is the number of discoveries + // Therefore, this error is not relevant, if raised + log.Entry().Warn("There are findings to review") + } + + // 4: Export report in workspace + reports := []piperutils.Path{} + reports = append(reports, piperutils.Path{Target: fmt.Sprintf("%v", piperReportName)}) + piperutils.PersistReportsAndLinks("credentialdiggerScan", "./", utils, reports, nil) + + return nil +} + +func executeCredentialDiggerProcess(utils credentialdiggerUtils, args []string) error { + return utils.RunExecutable("credentialdigger", args...) +} + +// hasConfigurationFile checks if the given file exists +func hasRulesFile(file string, utils credentialdiggerUtils) bool { + exists, err := utils.FileExists(file) + if err != nil { + log.Entry().WithError(err).Error() + } + return exists +} + +func credentialdiggerAddRules(config *credentialdiggerScanOptions, telemetryData *telemetry.CustomData, service credentialdiggerUtils) error { + // Credentialdigger home can be changed with local forks (e.g., for local piper runs) + cdHome := "/credential-digger-ui" // cdHome path as in docker container + if cdh := os.Getenv("CREDENTIALDIGGER_HOME"); cdh != "" { + cdHome = cdh + } + log.Entry().Debug("Use credentialdigger home ", cdHome) + // Set the rule file to the standard ruleset shipped within credential + // digger container + ruleFile := filepath.Join(cdHome, "backend", "rules.yml") + + if config.RulesDownloadURL != "" { + // Download custom rule file from this URL + log.Entry().Debugf("Download custom ruleset from %v", config.RulesDownloadURL) + dlClient := piperhttp.Client{} + ruleFile := filepath.Join(cdHome, "backend", "custom-rules.yml") + dlClient.DownloadFile(config.RulesDownloadURL, ruleFile, nil, nil) + log.Entry().Info("Download and use remote rules") + } else { + log.Entry().Debug("Use a local ruleset") + // Use rules defined in stashed file + if hasRulesFile(config.RulesFile, service) { + log.Entry().WithField("file", config.RulesFile).Info("Use stashed rules file from repository") + ruleFile = config.RulesFile + } else { + log.Entry().Info("Use standard pre-defined rules") + } + + } + cmd_list := []string{"add_rules", "--sqlite", piperDbName, ruleFile} + return executeCredentialDiggerProcess(service, cmd_list) +} + +func credentialdiggerGetDiscoveries(config *credentialdiggerScanOptions, telemetryData *telemetry.CustomData, service credentialdiggerUtils) error { + log.Entry().Info("Get discoveries") + cmd_list := []string{"get_discoveries", config.Repository, "--sqlite", piperDbName, + "--save", piperReportName} + // Export all the discoveries or export only new ones + if !config.ExportAll { + cmd_list = append(cmd_list, "--state", "new") + } + err := executeCredentialDiggerProcess(service, cmd_list) + if err != nil { + log.Entry().Error("credentialdiggerScan: Failed running credentialdigger get_discoveries") + log.Entry().Error(err) + return err + } + log.Entry().Info("Scan complete") + return nil +} + +func credentialdiggerBuildCommonArgs(config *credentialdiggerScanOptions) []string { + /*Some arguments are the same for all the scan flavors. Build them here + * not to duplicate code.*/ + scan_args := []string{} + // Repository url and sqlite db (always mandatory) + scan_args = append(scan_args, config.Repository, "--sqlite", piperDbName) + //git token is not mandatory for base credential digger tool, but in + //piper it is + scan_args = append(scan_args, "--git_token", config.Token) + //debug + if config.Debug { + log.Entry().Debug("Run the scan in debug mode") + scan_args = append(scan_args, "--debug") + } + //models + if len(config.Models) > 0 { + log.Entry().Debugf("Enable models %v", config.Models) + scan_args = append(scan_args, "--models") + scan_args = append(scan_args, config.Models...) + } + + return scan_args +} + +func credentialdiggerScanSnapshot(config *credentialdiggerScanOptions, telemetryData *telemetry.CustomData, service credentialdiggerUtils) error { + log.Entry().Infof("Scan Snapshot %v from repo %v", config.Snapshot, config.Repository) + cmd_list := []string{"scan_snapshot", + "--snapshot", config.Snapshot} + cmd_list = append(cmd_list, credentialdiggerBuildCommonArgs(config)...) + leaks := executeCredentialDiggerProcess(service, cmd_list) + if leaks != nil { + log.Entry().Warn("The scan found potential leaks in this Snapshot") + return leaks + } else { + log.Entry().Info("No leaks found") + return nil + } +} + +func credentialdiggerScanPR(config *credentialdiggerScanOptions, telemetryData *telemetry.CustomData, service credentialdiggerUtils) error { + log.Entry().Infof("Scan PR %v from repo %v", config.PrNumber, config.Repository) + cmd_list := []string{"scan_pr", + "--pr", strconv.Itoa(config.PrNumber), + "--api_endpoint", config.APIURL} + cmd_list = append(cmd_list, credentialdiggerBuildCommonArgs(config)...) + leaks := executeCredentialDiggerProcess(service, cmd_list) + if leaks != nil { + log.Entry().Warn("The scan found potential leaks in this PR") + return leaks + } else { + log.Entry().Info("No leaks found") + return nil + } +} + +func credentialdiggerFullScan(config *credentialdiggerScanOptions, telemetryData *telemetry.CustomData, service credentialdiggerUtils) error { + log.Entry().Infof("Full scan of repository %v", config.Repository) + cmd_list := []string{"scan"} + cmd_list = append(cmd_list, credentialdiggerBuildCommonArgs(config)...) + leaks := executeCredentialDiggerProcess(service, cmd_list) + if leaks != nil { + log.Entry().Warn("The scan found potential leaks") + log.Entry().Warnf("%v potential leaks found", leaks) + return leaks + } else { + log.Entry().Info("No leaks found") + return nil + } +} diff --git a/cmd/credentialdiggerScan_generated.go b/cmd/credentialdiggerScan_generated.go new file mode 100644 index 000000000..02e116744 --- /dev/null +++ b/cmd/credentialdiggerScan_generated.go @@ -0,0 +1,277 @@ +// Code generated by piper's step-generator. DO NOT EDIT. + +package cmd + +import ( + "fmt" + "os" + "time" + + "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/splunk" + "github.com/SAP/jenkins-library/pkg/telemetry" + "github.com/SAP/jenkins-library/pkg/validation" + "github.com/spf13/cobra" +) + +type credentialdiggerScanOptions struct { + Repository string `json:"repository,omitempty"` + Snapshot string `json:"snapshot,omitempty"` + PrNumber int `json:"prNumber,omitempty"` + ExportAll bool `json:"exportAll,omitempty"` + APIURL string `json:"apiUrl,omitempty"` + Debug bool `json:"debug,omitempty"` + RulesDownloadURL string `json:"rulesDownloadUrl,omitempty"` + Models []string `json:"models,omitempty"` + Token string `json:"token,omitempty"` + RulesFile string `json:"rulesFile,omitempty"` +} + +// CredentialdiggerScanCommand Scan a repository on GitHub with Credential Digger +func CredentialdiggerScanCommand() *cobra.Command { + const STEP_NAME = "credentialdiggerScan" + + metadata := credentialdiggerScanMetadata() + var stepConfig credentialdiggerScanOptions + var startTime time.Time + var logCollector *log.CollectorHook + var splunkClient *splunk.Splunk + telemetryClient := &telemetry.Telemetry{} + + var createCredentialdiggerScanCmd = &cobra.Command{ + Use: STEP_NAME, + Short: "Scan a repository on GitHub with Credential Digger", + Long: `This step allows you to scan a repository on Github using Credential Digger. + +It can for example be used for DevSecOps scenarios to verify the source code does not contain hard-coded credentials before being merged or released for production. +It supports several scan flavors, i.e., full scans of a repo, scan of a snapshot, or scan of a pull request.`, + PreRunE: func(cmd *cobra.Command, _ []string) error { + startTime = time.Now() + log.SetStepName(STEP_NAME) + log.SetVerbose(GeneralConfig.Verbose) + + GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) + + path, _ := os.Getwd() + fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} + log.RegisterHook(fatalHook) + + err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + if err != nil { + log.SetErrorCategory(log.ErrorConfiguration) + return err + } + log.RegisterSecret(stepConfig.Token) + + if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 { + sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID) + log.RegisterHook(&sentryHook) + } + + if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 { + splunkClient = &splunk.Splunk{} + logCollector = &log.CollectorHook{CorrelationID: GeneralConfig.CorrelationID} + log.RegisterHook(logCollector) + } + + if err = log.RegisterANSHookIfConfigured(GeneralConfig.CorrelationID); err != nil { + log.Entry().WithError(err).Warn("failed to set up SAP Alert Notification Service log hook") + } + + validation, err := validation.New(validation.WithJSONNamesForStructFields(), validation.WithPredefinedErrorMessages()) + if err != nil { + return err + } + if err = validation.ValidateStruct(stepConfig); err != nil { + log.SetErrorCategory(log.ErrorConfiguration) + return err + } + + return nil + }, + Run: func(_ *cobra.Command, _ []string) { + stepTelemetryData := telemetry.CustomData{} + stepTelemetryData.ErrorCode = "1" + handler := func() { + config.RemoveVaultSecretFiles() + stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) + stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() + stepTelemetryData.PiperCommitHash = GitCommit + telemetryClient.SetData(&stepTelemetryData) + telemetryClient.Send() + if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 { + splunkClient.Send(telemetryClient.GetData(), logCollector) + } + } + log.DeferExitHandler(handler) + defer handler() + telemetryClient.Initialize(GeneralConfig.NoTelemetry, STEP_NAME) + if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 { + splunkClient.Initialize(GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.SplunkConfig.Dsn, + GeneralConfig.HookConfig.SplunkConfig.Token, + GeneralConfig.HookConfig.SplunkConfig.Index, + GeneralConfig.HookConfig.SplunkConfig.SendLogs) + } + credentialdiggerScan(stepConfig, &stepTelemetryData) + stepTelemetryData.ErrorCode = "0" + log.Entry().Info("SUCCESS") + }, + } + + addCredentialdiggerScanFlags(createCredentialdiggerScanCmd, &stepConfig) + return createCredentialdiggerScanCmd +} + +func addCredentialdiggerScanFlags(cmd *cobra.Command, stepConfig *credentialdiggerScanOptions) { + cmd.Flags().StringVar(&stepConfig.Repository, "repository", os.Getenv("PIPER_repository"), "URL of the GitHub repository (was name, but we need the url). In case it's missing, use the URL of the current repository.") + cmd.Flags().StringVar(&stepConfig.Snapshot, "snapshot", os.Getenv("PIPER_snapshot"), "If set, scan the snapshot of the repository at this commit_id/branch.") + cmd.Flags().IntVar(&stepConfig.PrNumber, "prNumber", 0, "If set, scan the pull request open with this number.") + cmd.Flags().BoolVar(&stepConfig.ExportAll, "exportAll", false, "Export all the findings, i.e., including non-leaks.") + cmd.Flags().StringVar(&stepConfig.APIURL, "apiUrl", `https://api.github.com`, "Set the GitHub API url. Needed for scanning a pull request.") + cmd.Flags().BoolVar(&stepConfig.Debug, "debug", false, "Execute the scans in debug mode (i.e., print logs).") + cmd.Flags().StringVar(&stepConfig.RulesDownloadURL, "rulesDownloadUrl", os.Getenv("PIPER_rulesDownloadUrl"), "URL where to download custom rules. The file published at this URL must be formatted as the default ruleset https://raw.githubusercontent.com/SAP/credential-digger/main/ui/backend/rules.yml") + cmd.Flags().StringSliceVar(&stepConfig.Models, "models", []string{}, "Machine learning models to automatically verify the findings.") + cmd.Flags().StringVar(&stepConfig.Token, "token", os.Getenv("PIPER_token"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line") + cmd.Flags().StringVar(&stepConfig.RulesFile, "rulesFile", `inputs/rules.yml`, "Name of the rules file used locally within the step. If a remote files for rules is declared as `rulesDownloadUrl`, the stashed file is ignored. If you change the file's name make sure your stashing configuration also reflects this.") + + cmd.MarkFlagRequired("apiUrl") + cmd.MarkFlagRequired("token") +} + +// retrieve step metadata +func credentialdiggerScanMetadata() config.StepData { + var theMetaData = config.StepData{ + Metadata: config.StepMetadata{ + Name: "credentialdiggerScan", + Aliases: []config.Alias{}, + Description: "Scan a repository on GitHub with Credential Digger", + }, + Spec: config.StepSpec{ + Inputs: config.StepInputs{ + Secrets: []config.StepSecrets{ + {Name: "githubTokenCredentialsId", Description: "Jenkins 'Secret text' credentials ID containing token to authenticate to GitHub.", Type: "jenkins"}, + }, + Parameters: []config.StepParameters{ + { + Name: "repository", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{{Name: "githubRepo"}}, + Default: os.Getenv("PIPER_repository"), + }, + { + Name: "snapshot", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_snapshot"), + }, + { + Name: "prNumber", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "int", + Mandatory: false, + Aliases: []config.Alias{}, + Default: 0, + }, + { + Name: "exportAll", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{}, + Default: false, + }, + { + Name: "apiUrl", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: true, + Aliases: []config.Alias{{Name: "githubApiUrl"}}, + Default: `https://api.github.com`, + }, + { + Name: "debug", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{{Name: "verbose"}}, + Default: false, + }, + { + Name: "rulesDownloadUrl", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_rulesDownloadUrl"), + }, + { + Name: "models", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: []string{}, + }, + { + Name: "token", + ResourceRef: []config.ResourceReference{ + { + Name: "githubTokenCredentialsId", + Type: "secret", + }, + + { + Name: "githubVaultSecretName", + Type: "vaultSecret", + Default: "github", + }, + }, + Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: true, + Aliases: []config.Alias{{Name: "githubToken"}, {Name: "access_token"}}, + Default: os.Getenv("PIPER_token"), + }, + { + Name: "rulesFile", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: `inputs/rules.yml`, + }, + }, + }, + Containers: []config.Container{ + {Image: "credentialdigger.int.repositories.cloud.sap/credential_digger:4.9.2"}, + }, + Outputs: config.StepOutputs{ + Resources: []config.StepResources{ + { + Name: "report", + Type: "report", + Parameters: []map[string]interface{}{ + {"filePattern": "**/report*.csv", "type": "credentialdigger-report"}, + }, + }, + }, + }, + }, + } + return theMetaData +} diff --git a/cmd/credentialdiggerScan_generated_test.go b/cmd/credentialdiggerScan_generated_test.go new file mode 100644 index 000000000..646c3a9ac --- /dev/null +++ b/cmd/credentialdiggerScan_generated_test.go @@ -0,0 +1,17 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCredentialdiggerScanCommand(t *testing.T) { + t.Parallel() + + testCmd := CredentialdiggerScanCommand() + + // only high level testing performed - details are tested in step generation procedure + assert.Equal(t, "credentialdiggerScan", testCmd.Use, "command name incorrect") + +} diff --git a/cmd/credentialdiggerScan_test.go b/cmd/credentialdiggerScan_test.go new file mode 100644 index 000000000..ae5fa85d7 --- /dev/null +++ b/cmd/credentialdiggerScan_test.go @@ -0,0 +1,124 @@ +package cmd + +import ( + "errors" + "testing" + + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/stretchr/testify/assert" +) + +type credentialdiggerScanMockUtils struct { + *mock.ExecMockRunner + *mock.FilesMock + noerr bool +} + +func newCDTestsUtils() credentialdiggerScanMockUtils { + utils := credentialdiggerScanMockUtils{ + ExecMockRunner: &mock.ExecMockRunner{}, + FilesMock: &mock.FilesMock{}, + noerr: true, // flag for return value of MockRunner + } + return utils +} +func (c credentialdiggerScanMockUtils) RunExecutable(executable string, params ...string) error { + if c.noerr { + return nil + } else { + return errors.New("Some custom error") + } +} + +func TestCredentialdiggerFullScan(t *testing.T) { + t.Run("Valid full scan without discoveries", func(t *testing.T) { + config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken"} + utils := newCDTestsUtils() + assert.Equal(t, nil, credentialdiggerFullScan(&config, nil, utils)) + + }) + t.Run("Full scan with discoveries or wrong arguments", func(t *testing.T) { + config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken"} + utils := newCDTestsUtils() + utils.noerr = false + assert.EqualError(t, credentialdiggerFullScan(&config, nil, utils), "Some custom error") + }) +} + +func TestCredentialdiggerScanSnapshot(t *testing.T) { + t.Run("Valid scan snapshot without discoveries", func(t *testing.T) { + config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken", Snapshot: "main"} + utils := newCDTestsUtils() + assert.Equal(t, nil, credentialdiggerScanSnapshot(&config, nil, utils)) + }) + t.Run("Scan snapshot with discoveries or wrong arguments", func(t *testing.T) { + config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken", Snapshot: "main"} + utils := newCDTestsUtils() + utils.noerr = false + assert.EqualError(t, credentialdiggerScanSnapshot(&config, nil, utils), "Some custom error") + }) +} + +func TestCredentialdiggerScanPR(t *testing.T) { + t.Run("Valid scan pull request without discoveries", func(t *testing.T) { + config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken", PrNumber: 1} + utils := newCDTestsUtils() + assert.Equal(t, nil, credentialdiggerScanPR(&config, nil, utils)) + }) + t.Run("Scan pull request with discoveries or wrong arguments", func(t *testing.T) { + config := credentialdiggerScanOptions{Repository: "testRepo", Token: "validToken", PrNumber: 1} + utils := newCDTestsUtils() + utils.noerr = false + assert.EqualError(t, credentialdiggerScanPR(&config, nil, utils), "Some custom error") + }) +} + +func TestCredentialdiggerAddRules(t *testing.T) { + t.Run("Valid standard or remote rules", func(t *testing.T) { + config := credentialdiggerScanOptions{} + utils := newCDTestsUtils() + assert.Equal(t, nil, credentialdiggerAddRules(&config, nil, utils)) + }) + t.Run("Broken add rules", func(t *testing.T) { + config := credentialdiggerScanOptions{} + utils := newCDTestsUtils() + utils.noerr = false + assert.EqualError(t, credentialdiggerAddRules(&config, nil, utils), "Some custom error") + }) + /* + // In case we want to test the error raised by piperhttp + t.Run("Invalid external rules link", func(t *testing.T) { + rulesExt := "https://broken-link.com/fakerules" + config := credentialdiggerScanOptions{RulesDownloadURL: rulesExt} + utils := newCDTestsUtils() + assert.Equal(t, nil, credentialdiggerAddRules(&config, nil, utils)) + }) + */ +} + +func TestCredentialdiggerGetDiscoveries(t *testing.T) { + t.Run("Empty discoveries", func(t *testing.T) { + config := credentialdiggerScanOptions{Repository: "testRepo"} + utils := newCDTestsUtils() + assert.Equal(t, nil, credentialdiggerGetDiscoveries(&config, nil, utils)) + }) + t.Run("Get discoveries non-empty", func(t *testing.T) { + config := credentialdiggerScanOptions{Repository: "testRepo"} + utils := newCDTestsUtils() + utils.noerr = false + assert.EqualError(t, credentialdiggerGetDiscoveries(&config, nil, utils), "Some custom error") + }) +} + +func TestCredentialdiggerBuildCommonArgs(t *testing.T) { + t.Run("Valid build common args", func(t *testing.T) { + arguments := []string{"repoURL", "--sqlite", "piper_step_db.db", "--git_token", "validToken", + "--debug", "--models", "model1", "model2"} + config := credentialdiggerScanOptions{Repository: "repoURL", Token: "validToken", Snapshot: "main", + Debug: true, PrNumber: 1, + Models: []string{"model1", "model2"}, + } + assert.Equal(t, arguments, credentialdiggerBuildCommonArgs(&config)) + }) + +} diff --git a/cmd/metadata_generated.go b/cmd/metadata_generated.go index 2dc26c5ec..56f8078e6 100644 --- a/cmd/metadata_generated.go +++ b/cmd/metadata_generated.go @@ -49,6 +49,7 @@ func GetAllStepMetadata() map[string]config.StepData { "codeqlExecuteScan": codeqlExecuteScanMetadata(), "containerExecuteStructureTests": containerExecuteStructureTestsMetadata(), "containerSaveImage": containerSaveImageMetadata(), + "credentialdiggerScan": credentialdiggerScanMetadata(), "detectExecuteScan": detectExecuteScanMetadata(), "fortifyExecuteScan": fortifyExecuteScanMetadata(), "gaugeExecuteTests": gaugeExecuteTestsMetadata(), diff --git a/cmd/piper.go b/cmd/piper.go index cd34b4e33..fda676578 100644 --- a/cmd/piper.go +++ b/cmd/piper.go @@ -113,6 +113,7 @@ func Execute() { rootCmd.AddCommand(CheckmarxExecuteScanCommand()) rootCmd.AddCommand(FortifyExecuteScanCommand()) rootCmd.AddCommand(CodeqlExecuteScanCommand()) + rootCmd.AddCommand(CredentialdiggerScanCommand()) rootCmd.AddCommand(MtaBuildCommand()) rootCmd.AddCommand(ProtecodeExecuteScanCommand()) rootCmd.AddCommand(MavenExecuteCommand()) diff --git a/documentation/docs/steps/credentialdiggerScan.md b/documentation/docs/steps/credentialdiggerScan.md new file mode 100644 index 000000000..63991c134 --- /dev/null +++ b/documentation/docs/steps/credentialdiggerScan.md @@ -0,0 +1,7 @@ +# ${docGenStepName} + +## ${docGenDescription} + +## ${docGenParameters} + +## ${docGenConfiguration} diff --git a/documentation/mkdocs.yml b/documentation/mkdocs.yml index a00df76e0..3b729edf2 100644 --- a/documentation/mkdocs.yml +++ b/documentation/mkdocs.yml @@ -94,6 +94,7 @@ nav: - commonPipelineEnvironment: steps/commonPipelineEnvironment.md - containerExecuteStructureTests: steps/containerExecuteStructureTests.md - containerPushToRegistry: steps/containerPushToRegistry.md + - credentialdiggerScan: steps/credentialdiggerScan.md - debugReportArchive: steps/debugReportArchive.md - detectExecuteScan: steps/detectExecuteScan.md - dockerExecute: steps/dockerExecute.md diff --git a/resources/metadata/credentialdiggerScan.yaml b/resources/metadata/credentialdiggerScan.yaml new file mode 100644 index 000000000..9ef2b696f --- /dev/null +++ b/resources/metadata/credentialdiggerScan.yaml @@ -0,0 +1,124 @@ +metadata: + name: credentialdiggerScan + description: Scan a repository on GitHub with Credential Digger + longDescription: | + This step allows you to scan a repository on Github using Credential Digger. + + It can for example be used for DevSecOps scenarios to verify the source code does not contain hard-coded credentials before being merged or released for production. + It supports several scan flavors, i.e., full scans of a repo, scan of a snapshot, or scan of a pull request. +spec: + inputs: + secrets: + - name: githubTokenCredentialsId + description: Jenkins 'Secret text' credentials ID containing token to authenticate to GitHub. + type: jenkins + params: + - name: repository + aliases: + - name: githubRepo + description: URL of the GitHub repository (was name, but we need the url). In case it's missing, use the URL of the current repository. + scope: + - PARAMETERS + - STAGES + - STEPS + type: string + mandatory: false + - name: snapshot + description: If set, scan the snapshot of the repository at this commit_id/branch. + scope: + - PARAMETERS + - STAGES + - STEPS + type: string + mandatory: false + - name: prNumber + description: If set, scan the pull request open with this number. + scope: + - PARAMETERS + - STAGES + - STEPS + type: int + mandatory: false + - name: exportAll + type: bool + description: Export all the findings, i.e., including non-leaks. + scope: + - PARAMETERS + - STAGES + - STEPS + default: false + - name: apiUrl + aliases: + - name: githubApiUrl + description: Set the GitHub API url. Needed for scanning a pull request. + scope: + - GENERAL + - PARAMETERS + - STAGES + - STEPS + type: string + default: https://api.github.com + mandatory: true + - name: debug + aliases: + - name: verbose + description: Execute the scans in debug mode (i.e., print logs). + scope: + - GENERAL + - PARAMETERS + - STAGES + - STEPS + type: bool + default: false + - name: rulesDownloadUrl + type: string + description: URL where to download custom rules. The file published at this URL must be formatted as the default ruleset https://raw.githubusercontent.com/SAP/credential-digger/main/ui/backend/rules.yml + scope: + - PARAMETERS + - STAGES + - STEPS + mandatory: false + - name: models + description: Machine learning models to automatically verify the findings. + scope: + - PARAMETERS + - STAGES + - STEPS + type: "[]string" + - name: token + aliases: + - name: githubToken + - name: access_token + description: GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line + scope: + - GENERAL + - PARAMETERS + - STAGES + - STEPS + type: string + mandatory: true + secret: true + resourceRef: + - name: githubTokenCredentialsId + type: secret + - type: vaultSecret + default: github + name: githubVaultSecretName + - name: rulesFile + type: string + description: Name of the rules file used locally within the step. If a remote files for rules is declared as `rulesDownloadUrl`, the stashed file is ignored. If you change the file's name make sure your stashing configuration also reflects this. + mandatory: false + scope: + - PARAMETERS + - STAGES + - STEPS + default: inputs/rules.yml + outputs: + resources: + - name: report + type: report + params: + - filePattern: "**/report*.csv" + type: credentialdigger-report + containers: + - image: "credentialdigger.int.repositories.cloud.sap/credential_digger:4.9.2" diff --git a/test/groovy/CommonStepsTest.groovy b/test/groovy/CommonStepsTest.groovy index a1800f8ae..84bfb2148 100644 --- a/test/groovy/CommonStepsTest.groovy +++ b/test/groovy/CommonStepsTest.groovy @@ -174,6 +174,7 @@ public class CommonStepsTest extends BasePiperTest{ 'gctsExecuteABAPUnitTests', //implementing new golang pattern without fields 'gctsCloneRepository', //implementing new golang pattern without fields 'codeqlExecuteScan', //implementing new golang pattern without fields + 'credentialdiggerScan', //implementing new golang pattern without fields 'fortifyExecuteScan', //implementing new golang pattern without fields 'gctsDeploy', //implementing new golang pattern without fields 'containerSaveImage', //implementing new golang pattern without fields diff --git a/vars/credentialdiggerScan.groovy b/vars/credentialdiggerScan.groovy new file mode 100644 index 000000000..996346283 --- /dev/null +++ b/vars/credentialdiggerScan.groovy @@ -0,0 +1,11 @@ +import groovy.transform.Field + +@Field String STEP_NAME = getClass().getName() +@Field String METADATA_FILE = 'metadata/credentialdiggerScan.yaml' + +void call(Map parameters = [:]) { + List credentials = [ + [type: 'token', id: 'githubTokenCredentialsId', env: ['PIPER_token']] + ] + piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials) +}