2020-02-21 13:56:09 +02:00
// Code generated by piper's step-generator. DO NOT EDIT.
2020-02-06 17:16:34 +02:00
package cmd
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/SAP/jenkins-library/pkg/config"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperenv"
2021-05-17 12:14:04 +02:00
"github.com/SAP/jenkins-library/pkg/splunk"
2020-02-06 17:16:34 +02:00
"github.com/SAP/jenkins-library/pkg/telemetry"
2021-10-01 12:49:05 +02:00
"github.com/SAP/jenkins-library/pkg/validation"
2020-02-06 17:16:34 +02:00
"github.com/spf13/cobra"
)
type protecodeExecuteScanOptions struct {
ExcludeCVEs string ` json:"excludeCVEs,omitempty" `
FailOnSevereVulnerabilities bool ` json:"failOnSevereVulnerabilities,omitempty" `
ScanImage string ` json:"scanImage,omitempty" `
DockerRegistryURL string ` json:"dockerRegistryUrl,omitempty" `
2020-08-12 14:57:11 +02:00
DockerConfigJSON string ` json:"dockerConfigJSON,omitempty" `
2021-11-15 13:06:48 +02:00
CleanupMode string ` json:"cleanupMode,omitempty" validate:"possible-values=none binary complete" `
2020-02-06 17:16:34 +02:00
FilePath string ` json:"filePath,omitempty" `
IncludeLayers bool ` json:"includeLayers,omitempty" `
TimeoutMinutes string ` json:"timeoutMinutes,omitempty" `
ServerURL string ` json:"serverUrl,omitempty" `
ReportFileName string ` json:"reportFileName,omitempty" `
FetchURL string ` json:"fetchUrl,omitempty" `
Group string ` json:"group,omitempty" `
2021-05-04 15:03:43 +02:00
VerifyOnly bool ` json:"verifyOnly,omitempty" `
2021-09-13 11:13:48 +02:00
ReplaceProductID int ` json:"replaceProductId,omitempty" `
2020-04-20 16:44:01 +02:00
Username string ` json:"username,omitempty" `
2020-02-06 17:16:34 +02:00
Password string ` json:"password,omitempty" `
2021-05-05 19:52:13 +02:00
Version string ` json:"version,omitempty" `
2020-02-06 17:16:34 +02:00
PullRequestName string ` json:"pullRequestName,omitempty" `
}
type protecodeExecuteScanInflux struct {
2021-03-18 11:32:03 +02:00
step_data struct {
fields struct {
protecode bool
}
tags struct {
}
}
2020-02-07 15:12:40 +02:00
protecode_data struct {
2020-02-06 17:16:34 +02:00
fields struct {
2021-03-10 17:00:53 +02:00
excluded_vulnerabilities int
historical_vulnerabilities int
major_vulnerabilities int
minor_vulnerabilities int
triaged_vulnerabilities int
vulnerabilities int
2020-02-06 17:16:34 +02:00
}
tags struct {
}
}
}
func ( i * protecodeExecuteScanInflux ) persist ( path , resourceName string ) {
measurementContent := [ ] struct {
measurement string
valType string
name string
2020-10-05 15:33:28 +02:00
value interface { }
2020-02-06 17:16:34 +02:00
} {
2021-03-18 11:32:03 +02:00
{ valType : config . InfluxField , measurement : "step_data" , name : "protecode" , value : i . step_data . fields . protecode } ,
2020-02-07 15:12:40 +02:00
{ valType : config . InfluxField , measurement : "protecode_data" , name : "excluded_vulnerabilities" , value : i . protecode_data . fields . excluded_vulnerabilities } ,
2021-03-10 17:00:53 +02:00
{ valType : config . InfluxField , measurement : "protecode_data" , name : "historical_vulnerabilities" , value : i . protecode_data . fields . historical_vulnerabilities } ,
2020-02-07 15:12:40 +02:00
{ valType : config . InfluxField , measurement : "protecode_data" , name : "major_vulnerabilities" , value : i . protecode_data . fields . major_vulnerabilities } ,
2021-03-10 17:00:53 +02:00
{ valType : config . InfluxField , measurement : "protecode_data" , name : "minor_vulnerabilities" , value : i . protecode_data . fields . minor_vulnerabilities } ,
{ valType : config . InfluxField , measurement : "protecode_data" , name : "triaged_vulnerabilities" , value : i . protecode_data . fields . triaged_vulnerabilities } ,
2020-02-07 15:12:40 +02:00
{ valType : config . InfluxField , measurement : "protecode_data" , name : "vulnerabilities" , value : i . protecode_data . fields . vulnerabilities } ,
2020-02-06 17:16:34 +02:00
}
errCount := 0
for _ , metric := range measurementContent {
err := piperenv . SetResourceParameter ( path , resourceName , filepath . Join ( metric . measurement , fmt . Sprintf ( "%vs" , metric . valType ) , metric . name ) , metric . value )
if err != nil {
log . Entry ( ) . WithError ( err ) . Error ( "Error persisting influx environment." )
errCount ++
}
}
if errCount > 0 {
2020-04-20 16:52:57 +02:00
log . Entry ( ) . Fatal ( "failed to persist Influx environment" )
2020-02-06 17:16:34 +02:00
}
}
2020-02-12 16:57:39 +02:00
// ProtecodeExecuteScanCommand Protecode is an Open Source Vulnerability Scanner that is capable of scanning binaries. It can be used to scan docker images but is supports many other programming languages especially those of the C family.
2020-02-06 17:16:34 +02:00
func ProtecodeExecuteScanCommand ( ) * cobra . Command {
2020-04-28 07:42:02 +02:00
const STEP_NAME = "protecodeExecuteScan"
2020-02-06 17:16:34 +02:00
metadata := protecodeExecuteScanMetadata ( )
var stepConfig protecodeExecuteScanOptions
var startTime time . Time
var influx protecodeExecuteScanInflux
2021-05-17 12:14:04 +02:00
var logCollector * log . CollectorHook
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
var splunkClient * splunk . Splunk
telemetryClient := & telemetry . Telemetry { }
2020-02-06 17:16:34 +02:00
var createProtecodeExecuteScanCmd = & cobra . Command {
2020-04-28 07:42:02 +02:00
Use : STEP_NAME ,
2020-02-12 16:57:39 +02:00
Short : "Protecode is an Open Source Vulnerability Scanner that is capable of scanning binaries. It can be used to scan docker images but is supports many other programming languages especially those of the C family." ,
Long : ` Protecode is an Open Source Vulnerability Scanner that is capable of scanning binaries . It can be used to scan docker images but is supports many other programming languages especially those of the C family .
2020-02-06 17:16:34 +02:00
! ! ! hint "Auditing findings (Triaging)"
Triaging is now supported by the Protecode backend and also Piper does consider this information during the analysis of the scan results though product versions are not supported by Protecode . Therefore please make sure that the ` + " ` " + `fileName` + " ` " + ` you are providing does either contain a stable version or that it does not contain one at all . By ensuring that you are able to triage CVEs globally on the upload file ' s name without affecting any other artifacts scanned in the same Protecode group and as such triaged vulnerabilities will be considered during the next scan and will not fail the build anymore . ` ,
2020-06-23 18:05:21 +02:00
PreRunE : func ( cmd * cobra . Command , _ [ ] string ) error {
2020-02-06 17:16:34 +02:00
startTime = time . Now ( )
2020-04-28 07:42:02 +02:00
log . SetStepName ( STEP_NAME )
2020-02-06 17:16:34 +02:00
log . SetVerbose ( GeneralConfig . Verbose )
2020-04-28 07:42:02 +02:00
2021-07-08 15:26:07 +02:00
GeneralConfig . GitHubAccessTokens = ResolveAccessTokens ( GeneralConfig . GitHubTokens )
2020-04-28 07:42:02 +02:00
path , _ := os . Getwd ( )
fatalHook := & log . FatalHook { CorrelationID : GeneralConfig . CorrelationID , Path : path }
log . RegisterHook ( fatalHook )
err := PrepareConfig ( cmd , & metadata , STEP_NAME , & stepConfig , config . OpenPiperFile )
2020-04-16 14:37:45 +02:00
if err != nil {
2020-06-24 10:04:05 +02:00
log . SetErrorCategory ( log . ErrorConfiguration )
2020-04-16 14:37:45 +02:00
return err
}
2020-08-12 14:57:11 +02:00
log . RegisterSecret ( stepConfig . DockerConfigJSON )
2020-04-20 16:44:01 +02:00
log . RegisterSecret ( stepConfig . Username )
2020-04-16 14:37:45 +02:00
log . RegisterSecret ( stepConfig . Password )
2020-05-05 08:36:24 +02:00
if len ( GeneralConfig . HookConfig . SentryConfig . Dsn ) > 0 {
sentryHook := log . NewSentryHook ( GeneralConfig . HookConfig . SentryConfig . Dsn , GeneralConfig . CorrelationID )
log . RegisterHook ( & sentryHook )
}
2021-05-17 12:14:04 +02:00
if len ( GeneralConfig . HookConfig . SplunkConfig . Dsn ) > 0 {
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
splunkClient = & splunk . Splunk { }
2021-05-17 12:14:04 +02:00
logCollector = & log . CollectorHook { CorrelationID : GeneralConfig . CorrelationID }
log . RegisterHook ( logCollector )
}
2021-10-01 12:49:05 +02:00
validation , err := validation . New ( validation . WithJSONNamesForStructFields ( ) , validation . WithPredefinedErrorMessages ( ) )
if err != nil {
return err
}
if err = validation . ValidateStruct ( stepConfig ) ; err != nil {
log . SetErrorCategory ( log . ErrorConfiguration )
return err
}
2020-04-16 14:37:45 +02:00
return nil
2020-02-06 17:16:34 +02:00
} ,
2020-06-23 18:05:21 +02:00
Run : func ( _ * cobra . Command , _ [ ] string ) {
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
stepTelemetryData := telemetry . CustomData { }
stepTelemetryData . ErrorCode = "1"
2020-02-06 17:16:34 +02:00
handler := func ( ) {
2020-10-26 15:20:04 +02:00
config . RemoveVaultSecretFiles ( )
2020-02-06 17:16:34 +02:00
influx . persist ( GeneralConfig . EnvRootPath , "influx" )
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
stepTelemetryData . Duration = fmt . Sprintf ( "%v" , time . Since ( startTime ) . Milliseconds ( ) )
stepTelemetryData . ErrorCategory = log . GetErrorCategory ( ) . String ( )
stepTelemetryData . PiperCommitHash = GitCommit
telemetryClient . SetData ( & stepTelemetryData )
telemetryClient . Send ( )
2021-05-17 12:14:04 +02:00
if len ( GeneralConfig . HookConfig . SplunkConfig . Dsn ) > 0 {
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
splunkClient . Send ( telemetryClient . GetData ( ) , logCollector )
2021-05-17 12:14:04 +02:00
}
2020-02-06 17:16:34 +02:00
}
log . DeferExitHandler ( handler )
defer handler ( )
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
telemetryClient . Initialize ( GeneralConfig . NoTelemetry , STEP_NAME )
2021-05-17 12:14:04 +02:00
if len ( GeneralConfig . HookConfig . SplunkConfig . Dsn ) > 0 {
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
splunkClient . Initialize ( GeneralConfig . CorrelationID ,
2021-05-17 12:14:04 +02:00
GeneralConfig . HookConfig . SplunkConfig . Dsn ,
GeneralConfig . HookConfig . SplunkConfig . Token ,
GeneralConfig . HookConfig . SplunkConfig . Index ,
GeneralConfig . HookConfig . SplunkConfig . SendLogs )
}
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
protecodeExecuteScan ( stepConfig , & stepTelemetryData , & influx )
stepTelemetryData . ErrorCode = "0"
2020-05-18 21:52:54 +02:00
log . Entry ( ) . Info ( "SUCCESS" )
2020-02-06 17:16:34 +02:00
} ,
}
addProtecodeExecuteScanFlags ( createProtecodeExecuteScanCmd , & stepConfig )
return createProtecodeExecuteScanCmd
}
func addProtecodeExecuteScanFlags ( cmd * cobra . Command , stepConfig * protecodeExecuteScanOptions ) {
2020-09-17 21:05:03 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . ExcludeCVEs , "excludeCVEs" , ` ` , "DEPRECATED: Do use triaging within the Protecode UI instead" )
2020-02-06 17:16:34 +02:00
cmd . Flags ( ) . BoolVar ( & stepConfig . FailOnSevereVulnerabilities , "failOnSevereVulnerabilities" , true , "Whether to fail the job on severe vulnerabilties or not" )
cmd . Flags ( ) . StringVar ( & stepConfig . ScanImage , "scanImage" , os . Getenv ( "PIPER_scanImage" ) , "The reference to the docker image to scan with Protecode" )
cmd . Flags ( ) . StringVar ( & stepConfig . DockerRegistryURL , "dockerRegistryUrl" , os . Getenv ( "PIPER_dockerRegistryUrl" ) , "The reference to the docker registry to scan with Protecode" )
2020-08-12 14:57:11 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . DockerConfigJSON , "dockerConfigJSON" , os . Getenv ( "PIPER_dockerConfigJSON" ) , "Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/)." )
2020-05-25 19:48:59 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . CleanupMode , "cleanupMode" , ` binary ` , "Decides which parts are removed from the Protecode backend after the scan" )
2020-02-06 17:16:34 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . FilePath , "filePath" , os . Getenv ( "PIPER_filePath" ) , "The path to the file from local workspace to scan with Protecode" )
cmd . Flags ( ) . BoolVar ( & stepConfig . IncludeLayers , "includeLayers" , false , "Flag if the docker layers should be included" )
2020-05-25 19:48:59 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . TimeoutMinutes , "timeoutMinutes" , ` 60 ` , "The timeout to wait for the scan to finish" )
2020-02-06 17:16:34 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . ServerURL , "serverUrl" , os . Getenv ( "PIPER_serverUrl" ) , "The URL to the Protecode backend" )
2020-05-25 19:48:59 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . ReportFileName , "reportFileName" , ` protecode_report.pdf ` , "The file name of the report to be created" )
2021-03-12 10:58:21 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . FetchURL , "fetchUrl" , os . Getenv ( "PIPER_fetchUrl" ) , "The URL to fetch the file or image to scan with Protecode." )
2020-02-06 17:16:34 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . Group , "group" , os . Getenv ( "PIPER_group" ) , "The Protecode group ID of your team" )
2021-05-04 15:03:43 +02:00
cmd . Flags ( ) . BoolVar ( & stepConfig . VerifyOnly , "verifyOnly" , false , "Whether the step shall only apply verification checks or whether it does a full scan and check cycle" )
2021-10-21 10:03:42 +02:00
cmd . Flags ( ) . IntVar ( & stepConfig . ReplaceProductID , "replaceProductId" , 0 , "Specify <replaceProductId> which application binary will be replaced and rescanned and product id remains unchanged. By using this parameter, Protecode avoids creating multiple same products. Note this will affect results and feeds. If product id is not specified, then Piper starts auto detection mechanism, more precisely it searches a product id with scanned product name in that specified group, if there are several scans have been done with the same product name then the latest scan id will be fetched from BDBA backend. After obtaining product id, Piper re-uploads / replaces new binary without affecting already existing product id." )
2020-04-20 16:44:01 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . Username , "username" , os . Getenv ( "PIPER_username" ) , "User which is used for the protecode scan" )
2020-02-06 17:16:34 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . Password , "password" , os . Getenv ( "PIPER_password" ) , "Password which is used for the user" )
2021-05-05 19:52:13 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . Version , "version" , os . Getenv ( "PIPER_version" ) , "The version of the artifact to allow identification in protecode backend" )
2020-02-06 17:16:34 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . PullRequestName , "pullRequestName" , os . Getenv ( "PIPER_pullRequestName" ) , "The name of the pull request" )
cmd . MarkFlagRequired ( "serverUrl" )
cmd . MarkFlagRequired ( "group" )
2020-04-20 16:44:01 +02:00
cmd . MarkFlagRequired ( "username" )
2020-02-06 17:16:34 +02:00
cmd . MarkFlagRequired ( "password" )
}
// retrieve step metadata
func protecodeExecuteScanMetadata ( ) config . StepData {
var theMetaData = config . StepData {
2020-03-19 18:24:35 +02:00
Metadata : config . StepMetadata {
2020-11-20 09:13:59 +02:00
Name : "protecodeExecuteScan" ,
Aliases : [ ] config . Alias { } ,
Description : "Protecode is an Open Source Vulnerability Scanner that is capable of scanning binaries. It can be used to scan docker images but is supports many other programming languages especially those of the C family." ,
2020-03-19 18:24:35 +02:00
} ,
2020-02-06 17:16:34 +02:00
Spec : config . StepSpec {
Inputs : config . StepInputs {
2021-06-16 08:43:30 +02:00
Secrets : [ ] config . StepSecrets {
{ Name : "protecodeCredentialsId" , Description : "Jenkins 'Username with password' credentials ID containing username and password to authenticate to the Protecode system." , Type : "jenkins" } ,
2021-09-01 10:57:21 +02:00
{ Name : "dockerConfigJsonCredentialsId" , Description : "Jenkins 'Secret file' credentials ID containing Docker config.json (with registry credential(s)). You can create it like explained in [Prerequisites](https://www.project-piper.io/steps/protecodeExecuteScan/#prerequisites)." , Type : "jenkins" , Aliases : [ ] config . Alias { { Name : "dockerCredentialsId" , Deprecated : true } } } ,
2021-06-16 08:43:30 +02:00
} ,
2020-02-06 17:16:34 +02:00
Parameters : [ ] config . StepParameters {
{
Name : "excludeCVEs" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { { Name : "protecodeExcludeCVEs" } } ,
2021-06-16 08:43:30 +02:00
Default : ` ` ,
2020-02-06 17:16:34 +02:00
} ,
{
Name : "failOnSevereVulnerabilities" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "bool" ,
Mandatory : false ,
Aliases : [ ] config . Alias { { Name : "protecodeFailOnSevereVulnerabilities" } } ,
2021-06-16 08:43:30 +02:00
Default : true ,
2020-02-06 17:16:34 +02:00
} ,
{
2020-09-16 14:50:09 +02:00
Name : "scanImage" ,
ResourceRef : [ ] config . ResourceReference {
{
Name : "commonPipelineEnvironment" ,
Param : "container/imageNameTag" ,
} ,
} ,
Scope : [ ] string { "GENERAL" , "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { { Name : "dockerImage" } } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_scanImage" ) ,
2020-02-06 17:16:34 +02:00
} ,
{
2020-09-16 14:50:09 +02:00
Name : "dockerRegistryUrl" ,
ResourceRef : [ ] config . ResourceReference {
{
Name : "commonPipelineEnvironment" ,
Param : "container/registryUrl" ,
} ,
} ,
Scope : [ ] string { "GENERAL" , "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_dockerRegistryUrl" ) ,
2020-02-06 17:16:34 +02:00
} ,
2020-08-12 14:57:11 +02:00
{
2020-09-16 14:50:09 +02:00
Name : "dockerConfigJSON" ,
ResourceRef : [ ] config . ResourceReference {
2021-11-25 14:12:19 +02:00
{
Name : "commonPipelineEnvironment" ,
Param : "custom/dockerConfigJSON" ,
} ,
2020-09-16 14:50:09 +02:00
{
Name : "dockerConfigJsonCredentialsId" ,
Type : "secret" ,
} ,
2020-11-06 19:06:19 +02:00
{
2021-09-21 13:06:32 +02:00
Name : "dockerConfigFileVaultSecretName" ,
Type : "vaultSecretFile" ,
Default : "docker-config" ,
2020-11-06 19:06:19 +02:00
} ,
2020-09-16 14:50:09 +02:00
} ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_dockerConfigJSON" ) ,
2020-08-12 14:57:11 +02:00
} ,
2020-02-06 17:16:34 +02:00
{
Name : "cleanupMode" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : ` binary ` ,
2020-02-06 17:16:34 +02:00
} ,
{
Name : "filePath" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_filePath" ) ,
2020-02-06 17:16:34 +02:00
} ,
{
Name : "includeLayers" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "bool" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : false ,
2020-02-06 17:16:34 +02:00
} ,
{
Name : "timeoutMinutes" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { { Name : "protecodeTimeoutMinutes" } } ,
2021-06-16 08:43:30 +02:00
Default : ` 60 ` ,
2020-02-06 17:16:34 +02:00
} ,
{
Name : "serverUrl" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "GENERAL" , "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : true ,
Aliases : [ ] config . Alias { { Name : "protecodeServerUrl" } } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_serverUrl" ) ,
2020-02-06 17:16:34 +02:00
} ,
{
Name : "reportFileName" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : ` protecode_report.pdf ` ,
2020-02-06 17:16:34 +02:00
} ,
{
Name : "fetchUrl" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_fetchUrl" ) ,
2020-02-06 17:16:34 +02:00
} ,
{
Name : "group" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : true ,
Aliases : [ ] config . Alias { { Name : "protecodeGroup" } } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_group" ) ,
2020-02-06 17:16:34 +02:00
} ,
{
2021-05-04 15:03:43 +02:00
Name : "verifyOnly" ,
2020-02-06 17:16:34 +02:00
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "bool" ,
Mandatory : false ,
2021-11-23 16:37:28 +02:00
Aliases : [ ] config . Alias { { Name : "reuseExisting" , Deprecated : true } } ,
2021-06-16 08:43:30 +02:00
Default : false ,
2020-02-06 17:16:34 +02:00
} ,
2021-09-13 11:13:48 +02:00
{
Name : "replaceProductId" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "int" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
Default : 0 ,
} ,
2020-02-06 17:16:34 +02:00
{
2020-09-16 14:50:09 +02:00
Name : "username" ,
ResourceRef : [ ] config . ResourceReference {
{
Name : "protecodeCredentialsId" ,
Param : "username" ,
Type : "secret" ,
} ,
2020-12-22 18:43:57 +02:00
{
2021-09-21 13:06:32 +02:00
Name : "protecodeVaultSecretName" ,
Type : "vaultSecret" ,
Default : "protecode" ,
2020-12-22 18:43:57 +02:00
} ,
2020-09-16 14:50:09 +02:00
} ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : true ,
2021-11-23 16:37:28 +02:00
Aliases : [ ] config . Alias { { Name : "user" , Deprecated : true } } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_username" ) ,
2020-02-06 17:16:34 +02:00
} ,
{
2020-09-16 14:50:09 +02:00
Name : "password" ,
ResourceRef : [ ] config . ResourceReference {
{
Name : "protecodeCredentialsId" ,
Param : "password" ,
Type : "secret" ,
} ,
2020-12-22 18:43:57 +02:00
{
2021-09-21 13:06:32 +02:00
Name : "protecodeVaultSecretName" ,
Type : "vaultSecret" ,
Default : "protecode" ,
2020-12-22 18:43:57 +02:00
} ,
2020-09-16 14:50:09 +02:00
} ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : true ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_password" ) ,
2020-02-06 17:16:34 +02:00
} ,
{
2021-05-05 19:52:13 +02:00
Name : "version" ,
2020-09-16 14:50:09 +02:00
ResourceRef : [ ] config . ResourceReference {
{
Name : "commonPipelineEnvironment" ,
Param : "artifactVersion" ,
} ,
} ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
2021-11-23 16:37:28 +02:00
Aliases : [ ] config . Alias { { Name : "artifactVersion" , Deprecated : true } } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_version" ) ,
2020-02-06 17:16:34 +02:00
} ,
{
Name : "pullRequestName" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_pullRequestName" ) ,
2020-02-06 17:16:34 +02:00
} ,
} ,
} ,
2020-11-20 09:13:59 +02:00
Outputs : config . StepOutputs {
Resources : [ ] config . StepResources {
{
Name : "influx" ,
Type : "influx" ,
Parameters : [ ] map [ string ] interface { } {
2021-12-15 10:40:50 +02:00
{ "name" : "step_data" , "fields" : [ ] map [ string ] string { { "name" : "protecode" } } } ,
{ "name" : "protecode_data" , "fields" : [ ] map [ string ] string { { "name" : "excluded_vulnerabilities" } , { "name" : "historical_vulnerabilities" } , { "name" : "major_vulnerabilities" } , { "name" : "minor_vulnerabilities" } , { "name" : "triaged_vulnerabilities" } , { "name" : "vulnerabilities" } } } ,
2020-11-20 09:13:59 +02:00
} ,
} ,
} ,
} ,
2020-02-06 17:16:34 +02:00
} ,
}
return theMetaData
}