2020-07-10 08:07:59 +02:00
// Code generated by piper's step-generator. DO NOT EDIT.
package cmd
import (
"fmt"
"os"
2020-10-14 11:13:08 +02:00
"path/filepath"
2022-12-13 11:51:14 +02:00
"reflect"
"strings"
2020-07-10 08:07:59 +02:00
"time"
"github.com/SAP/jenkins-library/pkg/config"
2022-12-13 11:51:14 +02:00
"github.com/SAP/jenkins-library/pkg/gcs"
2020-07-10 08:07:59 +02:00
"github.com/SAP/jenkins-library/pkg/log"
2020-10-14 11:13:08 +02:00
"github.com/SAP/jenkins-library/pkg/piperenv"
2021-05-17 12:14:04 +02:00
"github.com/SAP/jenkins-library/pkg/splunk"
2020-07-10 08:07:59 +02:00
"github.com/SAP/jenkins-library/pkg/telemetry"
2021-10-01 12:49:05 +02:00
"github.com/SAP/jenkins-library/pkg/validation"
2022-12-13 11:51:14 +02:00
"github.com/bmatcuk/doublestar"
2020-07-10 08:07:59 +02:00
"github.com/spf13/cobra"
)
type kanikoExecuteOptions struct {
2022-02-07 08:58:41 +02:00
BuildOptions [ ] string ` json:"buildOptions,omitempty" `
BuildSettingsInfo string ` json:"buildSettingsInfo,omitempty" `
ContainerBuildOptions string ` json:"containerBuildOptions,omitempty" `
ContainerImage string ` json:"containerImage,omitempty" `
2022-08-15 09:05:20 +02:00
ContainerImageName string ` json:"containerImageName,omitempty" validate:"required_if=ContainerMultiImageBuild true" `
2022-02-07 08:58:41 +02:00
ContainerImageTag string ` json:"containerImageTag,omitempty" `
2022-03-11 10:47:44 +02:00
ContainerMultiImageBuild bool ` json:"containerMultiImageBuild,omitempty" `
ContainerMultiImageBuildExcludes [ ] string ` json:"containerMultiImageBuildExcludes,omitempty" `
ContainerMultiImageBuildTrimDir string ` json:"containerMultiImageBuildTrimDir,omitempty" `
2022-02-07 08:58:41 +02:00
ContainerPreparationCommand string ` json:"containerPreparationCommand,omitempty" `
ContainerRegistryURL string ` json:"containerRegistryUrl,omitempty" `
2022-07-15 08:40:33 +02:00
ContainerRegistryUser string ` json:"containerRegistryUser,omitempty" `
ContainerRegistryPassword string ` json:"containerRegistryPassword,omitempty" `
2022-02-07 08:58:41 +02:00
CustomTLSCertificateLinks [ ] string ` json:"customTlsCertificateLinks,omitempty" `
DockerConfigJSON string ` json:"dockerConfigJSON,omitempty" `
DockerfilePath string ` json:"dockerfilePath,omitempty" `
2022-02-09 11:17:34 +02:00
TargetArchitectures [ ] string ` json:"targetArchitectures,omitempty" `
2022-02-24 18:38:47 +02:00
ReadImageDigest bool ` json:"readImageDigest,omitempty" `
2022-11-07 15:27:05 +02:00
CreateBOM bool ` json:"createBOM,omitempty" `
2022-11-28 11:21:06 +02:00
SyftDownloadURL string ` json:"syftDownloadUrl,omitempty" `
2020-07-10 08:07:59 +02:00
}
2020-10-14 11:13:08 +02:00
type kanikoExecuteCommonPipelineEnvironment struct {
container struct {
2022-02-07 08:58:41 +02:00
registryURL string
imageNameTag string
2022-02-23 12:41:26 +02:00
imageDigest string
2022-02-07 08:58:41 +02:00
imageNames [ ] string
imageNameTags [ ] string
2022-02-23 12:41:26 +02:00
imageDigests [ ] string
2020-10-14 11:13:08 +02:00
}
2021-12-01 08:46:18 +02:00
custom struct {
buildSettingsInfo string
}
2020-10-14 11:13:08 +02:00
}
func ( p * kanikoExecuteCommonPipelineEnvironment ) persist ( path , resourceName string ) {
content := [ ] struct {
category string
name string
value interface { }
} {
{ category : "container" , name : "registryUrl" , value : p . container . registryURL } ,
{ category : "container" , name : "imageNameTag" , value : p . container . imageNameTag } ,
2022-02-23 12:41:26 +02:00
{ category : "container" , name : "imageDigest" , value : p . container . imageDigest } ,
2022-02-07 08:58:41 +02:00
{ category : "container" , name : "imageNames" , value : p . container . imageNames } ,
{ category : "container" , name : "imageNameTags" , value : p . container . imageNameTags } ,
2022-02-23 12:41:26 +02:00
{ category : "container" , name : "imageDigests" , value : p . container . imageDigests } ,
2021-12-01 08:46:18 +02:00
{ category : "custom" , name : "buildSettingsInfo" , value : p . custom . buildSettingsInfo } ,
2020-10-14 11:13:08 +02:00
}
errCount := 0
for _ , param := range content {
err := piperenv . SetResourceParameter ( path , resourceName , filepath . Join ( param . category , param . name ) , param . value )
if err != nil {
log . Entry ( ) . WithError ( err ) . Error ( "Error persisting piper environment." )
errCount ++
}
}
if errCount > 0 {
2021-12-15 15:26:23 +02:00
log . Entry ( ) . Error ( "failed to persist Piper environment" )
2020-10-14 11:13:08 +02:00
}
}
2022-12-13 11:51:14 +02:00
type kanikoExecuteReports struct {
}
func ( p * kanikoExecuteReports ) persist ( stepConfig kanikoExecuteOptions , gcpJsonKeyFilePath string , gcsBucketId string , gcsFolderPath string , gcsSubFolder string ) {
if gcsBucketId == "" {
log . Entry ( ) . Info ( "persisting reports to GCS is disabled, because gcsBucketId is empty" )
return
}
log . Entry ( ) . Info ( "Uploading reports to Google Cloud Storage..." )
content := [ ] gcs . ReportOutputParam {
{ FilePattern : "**/bom-*.xml" , ParamRef : "" , StepResultType : "sbom" } ,
}
envVars := [ ] gcs . EnvVar {
{ Name : "GOOGLE_APPLICATION_CREDENTIALS" , Value : gcpJsonKeyFilePath , Modified : false } ,
}
gcsClient , err := gcs . NewClient ( gcs . WithEnvVars ( envVars ) )
if err != nil {
log . Entry ( ) . Errorf ( "creation of GCS client failed: %v" , err )
return
}
defer gcsClient . Close ( )
structVal := reflect . ValueOf ( & stepConfig ) . Elem ( )
inputParameters := map [ string ] string { }
for i := 0 ; i < structVal . NumField ( ) ; i ++ {
field := structVal . Type ( ) . Field ( i )
if field . Type . String ( ) == "string" {
paramName := strings . Split ( field . Tag . Get ( "json" ) , "," )
paramValue , _ := structVal . Field ( i ) . Interface ( ) . ( string )
inputParameters [ paramName [ 0 ] ] = paramValue
}
}
if err := gcs . PersistReportsToGCS ( gcsClient , content , inputParameters , gcsFolderPath , gcsBucketId , gcsSubFolder , doublestar . Glob , os . Stat ) ; err != nil {
log . Entry ( ) . Errorf ( "failed to persist reports: %v" , err )
}
}
2020-07-10 08:07:59 +02:00
// KanikoExecuteCommand Executes a [Kaniko](https://github.com/GoogleContainerTools/kaniko) build for creating a Docker container.
func KanikoExecuteCommand ( ) * cobra . Command {
const STEP_NAME = "kanikoExecute"
metadata := kanikoExecuteMetadata ( )
var stepConfig kanikoExecuteOptions
var startTime time . Time
2020-10-14 11:13:08 +02:00
var commonPipelineEnvironment kanikoExecuteCommonPipelineEnvironment
2022-12-13 11:51:14 +02:00
var reports kanikoExecuteReports
2021-05-17 12:14:04 +02:00
var logCollector * log . CollectorHook
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
var splunkClient * splunk . Splunk
telemetryClient := & telemetry . Telemetry { }
2020-07-10 08:07:59 +02:00
var createKanikoExecuteCmd = & cobra . Command {
Use : STEP_NAME ,
Short : "Executes a [Kaniko](https://github.com/GoogleContainerTools/kaniko) build for creating a Docker container." ,
2022-02-07 08:58:41 +02:00
Long : ` Executes a [ Kaniko ] ( https : //github.com/GoogleContainerTools/kaniko) build for creating a Docker container.
# # # Building multiple container images
The step allows you to build multiple container images with one run .
This is suitable in case you need to create multiple images for one microservice , e . g . for testing .
All images will get the same "root" name and the same versioning . < br / >
* * Thus , this is not suitable to be used for a monorepo approach ! * * For monorepos you need to use a build tool natively capable to take care for monorepos
or implement a custom logic and for example execute this ` + " ` " + `kanikoExecute` + " ` " + ` step multiple times in your custom pipeline .
2022-03-11 10:47:44 +02:00
You can activate multiple builds using the parameter [ containerMultiImageBuild ] ( # containermultiimagebuild )
Behavior can be adapted using :
* [ containerMultiImageBuildExcludes ] ( # containermultiimagebuildexcludes ) for defining excludes
* [ containerMultiImageBuildTrimDir ] ( # containermultiimagebuildtrimdir ) for removing parent directory part from image name
Examples :
# # # # Multiple containers in sub directories
Configuration as follows :
` + " ` " + `` + " ` " + ` ` + " ` " + `
general :
containerImageName : myImage
steps :
kanikoExecute :
containerMultiImageBuild : true
` + " ` " + `` + " ` " + ` ` + " ` " + `
Following Dockerfiles are available in the repository :
* sub1 / Dockerfile
* sub2 / Dockerfile
Following final image names will be built :
* ` + " ` " + `myImage-sub1` + " ` " + `
* ` + " ` " + `myImage-sub2` + " ` " + `
# # # # Multiple containers in sub directories while trimming a directory part
Configuration as follows :
` + " ` " + `` + " ` " + ` ` + " ` " + `
general :
containerImageName : myImage
steps :
kanikoExecute :
containerMultiImageBuild : true
containerMultiImageBuildTrimDir : . ci
` + " ` " + `` + " ` " + ` ` + " ` " + `
2022-02-07 08:58:41 +02:00
2022-03-11 10:47:44 +02:00
Following Dockerfiles are available in the repository :
* . ci / sub1 / Dockerfile
* . ci / sub2 / Dockerfile
Following final image names will be built :
* ` + " ` " + `myImage-sub1` + " ` " + `
* ` + " ` " + `myImage-sub2` + " ` " + ` ` ,
2020-07-10 08:07:59 +02:00
PreRunE : func ( cmd * cobra . Command , _ [ ] string ) error {
startTime = time . Now ( )
log . SetStepName ( STEP_NAME )
log . SetVerbose ( GeneralConfig . Verbose )
2021-07-08 15:26:07 +02:00
GeneralConfig . GitHubAccessTokens = ResolveAccessTokens ( GeneralConfig . GitHubTokens )
2020-07-10 08:07:59 +02:00
path , _ := os . Getwd ( )
fatalHook := & log . FatalHook { CorrelationID : GeneralConfig . CorrelationID , Path : path }
log . RegisterHook ( fatalHook )
err := PrepareConfig ( cmd , & metadata , STEP_NAME , & stepConfig , config . OpenPiperFile )
if err != nil {
log . SetErrorCategory ( log . ErrorConfiguration )
return err
}
log . RegisterSecret ( stepConfig . DockerConfigJSON )
if len ( GeneralConfig . HookConfig . SentryConfig . Dsn ) > 0 {
sentryHook := log . NewSentryHook ( GeneralConfig . HookConfig . SentryConfig . Dsn , GeneralConfig . CorrelationID )
log . RegisterHook ( & sentryHook )
}
2021-05-17 12:14:04 +02:00
if len ( GeneralConfig . HookConfig . SplunkConfig . Dsn ) > 0 {
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
splunkClient = & splunk . Splunk { }
2021-05-17 12:14:04 +02:00
logCollector = & log . CollectorHook { CorrelationID : GeneralConfig . CorrelationID }
log . RegisterHook ( logCollector )
}
2022-06-22 13:31:17 +02:00
if err = log . RegisterANSHookIfConfigured ( GeneralConfig . CorrelationID ) ; err != nil {
log . Entry ( ) . WithError ( err ) . Warn ( "failed to set up SAP Alert Notification Service log hook" )
}
2021-05-17 12:14:04 +02:00
2021-10-01 12:49:05 +02:00
validation , err := validation . New ( validation . WithJSONNamesForStructFields ( ) , validation . WithPredefinedErrorMessages ( ) )
if err != nil {
return err
}
if err = validation . ValidateStruct ( stepConfig ) ; err != nil {
log . SetErrorCategory ( log . ErrorConfiguration )
return err
}
2020-07-10 08:07:59 +02:00
return nil
} ,
Run : func ( _ * cobra . Command , _ [ ] string ) {
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
stepTelemetryData := telemetry . CustomData { }
stepTelemetryData . ErrorCode = "1"
2020-07-10 08:07:59 +02:00
handler := func ( ) {
2020-10-14 11:13:08 +02:00
commonPipelineEnvironment . persist ( GeneralConfig . EnvRootPath , "commonPipelineEnvironment" )
2022-12-13 11:51:14 +02:00
reports . persist ( stepConfig , GeneralConfig . GCPJsonKeyFilePath , GeneralConfig . GCSBucketId , GeneralConfig . GCSFolderPath , GeneralConfig . GCSSubFolder )
2021-12-15 16:07:47 +02:00
config . RemoveVaultSecretFiles ( )
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
stepTelemetryData . Duration = fmt . Sprintf ( "%v" , time . Since ( startTime ) . Milliseconds ( ) )
stepTelemetryData . ErrorCategory = log . GetErrorCategory ( ) . String ( )
stepTelemetryData . PiperCommitHash = GitCommit
telemetryClient . SetData ( & stepTelemetryData )
telemetryClient . Send ( )
2021-05-17 12:14:04 +02:00
if len ( GeneralConfig . HookConfig . SplunkConfig . Dsn ) > 0 {
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
splunkClient . Send ( telemetryClient . GetData ( ) , logCollector )
2021-05-17 12:14:04 +02:00
}
2020-07-10 08:07:59 +02:00
}
log . DeferExitHandler ( handler )
defer handler ( )
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
telemetryClient . Initialize ( GeneralConfig . NoTelemetry , STEP_NAME )
2021-05-17 12:14:04 +02:00
if len ( GeneralConfig . HookConfig . SplunkConfig . Dsn ) > 0 {
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
splunkClient . Initialize ( GeneralConfig . CorrelationID ,
2021-05-17 12:14:04 +02:00
GeneralConfig . HookConfig . SplunkConfig . Dsn ,
GeneralConfig . HookConfig . SplunkConfig . Token ,
GeneralConfig . HookConfig . SplunkConfig . Index ,
GeneralConfig . HookConfig . SplunkConfig . SendLogs )
}
Changes for Pipeline Reporting (#3213)
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetLog() function to orchestrator
* Fixes BUILD_NUMBER env variable
* Fixes correct env var for JENKINS_HOME
* Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation
* Adds possibility to read errorJsons; updates splunk package for log files (WIP)
* Uncommenting dev code
* Adds GetRequest function which holds the response in memory (not saved to disk)
* Implements GetLog() function for ADO, adds function to read PipelineRuntime
* PAT has been revoked
* Changes http package, s.t. if password only is required basic auth works too
* Adds env variable for azure token, error handling in case of unauthenticated/nil response
* Adds logging output in case env variable can not be read and fallback variable needs to be used
* Adds usage of environment variables for auth, uses jenkins api
* Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function
* Adds initaliziation function for orchestrator authetnication
* Adds settings struct for orchestrator authentication
* Adds function to whole logfile to Splunk
* Struct for pipeline related telemetry information
* Increase messagebatch size to 10k
* Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk
* Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation
* Changes content type to txt
* Send telemetry independent of logfiles, increases amount of messages per file
* Adds JobURL for orchestrators and UnknownOrchestrator as fallback
* telemetry makes use of orchestrator specific information
* Adds orchestrator independent correlationID
* Adds custom fields for pipeline status
* go fmt
* Removes env var test - no env variables are read anymore
* Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized
* Removes Custom fields from telemetry as these can not be reflected in SWA
* Adds custom telemetry information (piperHash,..) to each step telemetry information
* Removes falltrough in case no orchestrator has been found
* Updates tests for orchestrator package
* Adds orchestrator import in generated files
* Updates generator files for internal library
* Adds orchestrator telemetry information to steps
* Updates generated files, fatalHook writes to cpe
* Go generate from master, go fmt
* Adds Custom Data field LastErrorCode
* Removes GetLog() test
* Update init_unix.go
* Update docker_integration_test_executor.go
* Update integration_api_cli_test.go
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Reverts go1.17 fmt formatting
* Renames customTelemetryData to stepTelemetryData
* Adjustments to orchestrator-package, cleanup, adds JobName
* Adjusts commonPipelineEnvironment path
* Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package
* Go fmt
* Changes path for errorDetails, adds debug information
* Removes custom fields from step, adds orchestrator, commithash to baseMetadata
* Adjusts tests for telemetry package
* Adds tests for orchestrator
* Updates generated files, initalization of splunk client only if its available in the config
* Fixes typo in helper go
* Update pkg/http/downloader.go
* Update pkg/http/downloader.go
* Update pkg/log/fatalHook.go
* Update fatalHook.go
* Update pkg/splunk/splunk.go
* Update pkg/telemetry/data.go
* Adds GetBuildStatus() and GetAPIInformation() to orchestrators
* error formatting
* Bugfix: dont send telemetry data if disabled, adjusts test
* go fmt
* Use correct error handling
* Update pkg/telemetry/telemetry.go
* Fixes telemetry disabled in the tests
* Fixes http tests
* Log fatal errors to logFile
* Adds CustomReportingConfig to hooks
* Cleanup comments in splunk package
* Adds possibility to send telemetry to custom endpoint
* Adds debug output for the payload
* Debug output for the payload as a string
* Adds test cases for changes in telemetry package
* go fmt
* Adds generated files for new step
* Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17
* Adjusts test for sonarExecuteScan
* Adjusts test for sonarExecuteScan
* Adds explanation for customreportingConfig
* Makes disableing of customSend more obvious
* Adds custom step reporting to each step, updates generated files, adjusts helper testdata
* fixes unit test wrong usage of logging
* Send pipeline data altough there has been no error, adjust test cases
* Reverts changes for customReporting
* Updates generated files, removes customReporting
* Removes writing errorDetails to CPE
* Reverts usage of customreporting
* go fmt
* reverts changes in http_test
* reverts changes in http_test
* Skips integration cnb test
Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
kanikoExecute ( stepConfig , & stepTelemetryData , & commonPipelineEnvironment )
stepTelemetryData . ErrorCode = "0"
2020-07-10 08:07:59 +02:00
log . Entry ( ) . Info ( "SUCCESS" )
} ,
}
addKanikoExecuteFlags ( createKanikoExecuteCmd , & stepConfig )
return createKanikoExecuteCmd
}
func addKanikoExecuteFlags ( cmd * cobra . Command , stepConfig * kanikoExecuteOptions ) {
2022-03-07 12:42:33 +02:00
cmd . Flags ( ) . StringSliceVar ( & stepConfig . BuildOptions , "buildOptions" , [ ] string { ` --skip-tls-verify-pull ` , ` --ignore-path=/workspace ` , ` --ignore-path=/busybox ` } , "Defines a list of build options for the [kaniko](https://github.com/GoogleContainerTools/kaniko) build." )
2022-02-07 08:58:41 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . BuildSettingsInfo , "buildSettingsInfo" , os . Getenv ( "PIPER_buildSettingsInfo" ) , "Build settings info is typically filled by the step automatically to create information about the build settings that were used during the mta build. This information is typically used for compliance related processes." )
2020-07-10 08:07:59 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . ContainerBuildOptions , "containerBuildOptions" , os . Getenv ( "PIPER_containerBuildOptions" ) , "Deprected, please use buildOptions. Defines the build options for the [kaniko](https://github.com/GoogleContainerTools/kaniko) build." )
cmd . Flags ( ) . StringVar ( & stepConfig . ContainerImage , "containerImage" , os . Getenv ( "PIPER_containerImage" ) , "Defines the full name of the Docker image to be created including registry, image name and tag like `my.docker.registry/path/myImageName:myTag`. If left empty, image will not be pushed." )
2020-10-14 11:13:08 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . ContainerImageName , "containerImageName" , os . Getenv ( "PIPER_containerImageName" ) , "Name of the container which will be built - will be used instead of parameter `containerImage`" )
cmd . Flags ( ) . StringVar ( & stepConfig . ContainerImageTag , "containerImageTag" , os . Getenv ( "PIPER_containerImageTag" ) , "Tag of the container which will be built - will be used instead of parameter `containerImage`" )
2022-03-11 10:47:44 +02:00
cmd . Flags ( ) . BoolVar ( & stepConfig . ContainerMultiImageBuild , "containerMultiImageBuild" , false , "Defines if multiple containers should be build. Dockerfiles are used using the pattern **/Dockerfile*. Excludes can be defined via [`containerMultiImageBuildExcludes`](#containermultiimagebuildexscludes)." )
cmd . Flags ( ) . StringSliceVar ( & stepConfig . ContainerMultiImageBuildExcludes , "containerMultiImageBuildExcludes" , [ ] string { } , "Defines a list of Dockerfile paths to exclude from the build when using [`containerMultiImageBuild`](#containermultiimagebuild)." )
cmd . Flags ( ) . StringVar ( & stepConfig . ContainerMultiImageBuildTrimDir , "containerMultiImageBuildTrimDir" , os . Getenv ( "PIPER_containerMultiImageBuildTrimDir" ) , "Defines a trailing directory part which should not be considered in the final image name." )
2020-07-10 08:07:59 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . ContainerPreparationCommand , "containerPreparationCommand" , ` rm -f /kaniko/.docker/config.json ` , "Defines the command to prepare the Kaniko container. By default the contained credentials are removed in order to allow anonymous access to container registries." )
2020-10-14 11:13:08 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . ContainerRegistryURL , "containerRegistryUrl" , os . Getenv ( "PIPER_containerRegistryUrl" ) , "http(s) url of the Container registry where the image should be pushed to - will be used instead of parameter `containerImage`" )
2022-07-15 08:40:33 +02:00
cmd . Flags ( ) . StringVar ( & stepConfig . ContainerRegistryUser , "containerRegistryUser" , os . Getenv ( "PIPER_containerRegistryUser" ) , "Username of the Container registry where the image should be pushed to - which will updated in a docker config json file. If a docker config json file is provided via parameter `dockerConfigJSON` , then the existing file will be enhanced" )
cmd . Flags ( ) . StringVar ( & stepConfig . ContainerRegistryPassword , "containerRegistryPassword" , os . Getenv ( "PIPER_containerRegistryPassword" ) , "Password of the Container registry where the image should be pushed to - which will updated in a docker config json file. If a docker config json file is provided via parameter `dockerConfigJSON` , then the existing file will be enhanced" )
2020-07-10 08:07:59 +02:00
cmd . Flags ( ) . StringSliceVar ( & stepConfig . CustomTLSCertificateLinks , "customTlsCertificateLinks" , [ ] string { } , "List containing download links of custom TLS certificates. This is required to ensure trusted connections to registries with custom certificates." )
cmd . Flags ( ) . StringVar ( & stepConfig . DockerConfigJSON , "dockerConfigJSON" , os . Getenv ( "PIPER_dockerConfigJSON" ) , "Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/)." )
cmd . Flags ( ) . StringVar ( & stepConfig . DockerfilePath , "dockerfilePath" , ` Dockerfile ` , "Defines the location of the Dockerfile relative to the Jenkins workspace." )
2022-02-09 11:17:34 +02:00
cmd . Flags ( ) . StringSliceVar ( & stepConfig . TargetArchitectures , "targetArchitectures" , [ ] string { ` ` } , "Defines the target architectures for which the build should run using OS and architecture separated by a comma. (EXPERIMENTAL)" )
2022-02-24 18:38:47 +02:00
cmd . Flags ( ) . BoolVar ( & stepConfig . ReadImageDigest , "readImageDigest" , false , "" )
2022-12-13 11:51:14 +02:00
cmd . Flags ( ) . BoolVar ( & stepConfig . CreateBOM , "createBOM" , false , "Creates the bill of materials (BOM) using Syft and stores it in a file in CycloneDX 1.4 format." )
cmd . Flags ( ) . StringVar ( & stepConfig . SyftDownloadURL , "syftDownloadUrl" , ` https://github.com/anchore/syft/releases/download/v0.62.3/syft_0.62.3_linux_amd64.tar.gz ` , "Specifies the download url of the Syft Linux amd64 tar binary file. This can be found at https://github.com/anchore/syft/releases/." )
2020-07-10 08:07:59 +02:00
}
// retrieve step metadata
func kanikoExecuteMetadata ( ) config . StepData {
var theMetaData = config . StepData {
Metadata : config . StepMetadata {
2020-11-20 09:13:59 +02:00
Name : "kanikoExecute" ,
Aliases : [ ] config . Alias { } ,
Description : "Executes a [Kaniko](https://github.com/GoogleContainerTools/kaniko) build for creating a Docker container." ,
2020-07-10 08:07:59 +02:00
} ,
Spec : config . StepSpec {
Inputs : config . StepInputs {
2021-06-16 08:43:30 +02:00
Secrets : [ ] config . StepSecrets {
2021-09-01 13:07:13 +02:00
{ Name : "dockerConfigJsonCredentialsId" , Description : "Jenkins 'Secret file' credentials ID containing Docker config.json (with registry credential(s)). You can create it like explained in the [protocodeExecuteScan Prerequisites section](https://www.project-piper.io/steps/protecodeExecuteScan/#prerequisites)." , Type : "jenkins" } ,
2021-06-16 08:43:30 +02:00
} ,
2020-07-10 08:07:59 +02:00
Parameters : [ ] config . StepParameters {
{
Name : "buildOptions" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "[]string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2022-03-07 12:42:33 +02:00
Default : [ ] string { ` --skip-tls-verify-pull ` , ` --ignore-path=/workspace ` , ` --ignore-path=/busybox ` } ,
2022-02-07 08:58:41 +02:00
} ,
{
Name : "buildSettingsInfo" ,
ResourceRef : [ ] config . ResourceReference {
{
Name : "commonPipelineEnvironment" ,
Param : "custom/buildSettingsInfo" ,
} ,
} ,
Scope : [ ] string { "STEPS" , "STAGES" , "PARAMETERS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
Default : os . Getenv ( "PIPER_buildSettingsInfo" ) ,
} ,
2020-07-10 08:07:59 +02:00
{
Name : "containerBuildOptions" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_containerBuildOptions" ) ,
2020-07-10 08:07:59 +02:00
} ,
{
2020-10-14 11:13:08 +02:00
Name : "containerImage" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
2021-11-23 16:37:28 +02:00
Aliases : [ ] config . Alias { { Name : "containerImageNameAndTag" , Deprecated : true } } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_containerImage" ) ,
2020-10-14 11:13:08 +02:00
} ,
{
Name : "containerImageName" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "GENERAL" , "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { { Name : "dockerImageName" } } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_containerImageName" ) ,
2020-10-14 11:13:08 +02:00
} ,
{
Name : "containerImageTag" ,
2020-09-16 14:50:09 +02:00
ResourceRef : [ ] config . ResourceReference {
{
Name : "commonPipelineEnvironment" ,
2020-10-14 11:13:08 +02:00
Param : "artifactVersion" ,
2020-09-16 14:50:09 +02:00
} ,
} ,
2020-10-14 11:13:08 +02:00
Scope : [ ] string { "GENERAL" , "PARAMETERS" , "STAGES" , "STEPS" } ,
2020-09-16 14:50:09 +02:00
Type : "string" ,
Mandatory : false ,
2020-10-14 11:13:08 +02:00
Aliases : [ ] config . Alias { { Name : "artifactVersion" } } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_containerImageTag" ) ,
2020-07-10 08:07:59 +02:00
} ,
2022-03-11 10:47:44 +02:00
{
Name : "containerMultiImageBuild" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "GENERAL" , "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "bool" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
Default : false ,
} ,
{
Name : "containerMultiImageBuildExcludes" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "GENERAL" , "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "[]string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
Default : [ ] string { } ,
} ,
{
Name : "containerMultiImageBuildTrimDir" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
Default : os . Getenv ( "PIPER_containerMultiImageBuildTrimDir" ) ,
} ,
2020-07-10 08:07:59 +02:00
{
Name : "containerPreparationCommand" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : ` rm -f /kaniko/.docker/config.json ` ,
2020-07-10 08:07:59 +02:00
} ,
2020-10-14 11:13:08 +02:00
{
Name : "containerRegistryUrl" ,
ResourceRef : [ ] config . ResourceReference {
{
Name : "commonPipelineEnvironment" ,
Param : "container/registryUrl" ,
} ,
} ,
Scope : [ ] string { "GENERAL" , "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { { Name : "dockerRegistryUrl" } } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_containerRegistryUrl" ) ,
2020-10-14 11:13:08 +02:00
} ,
2022-07-15 08:40:33 +02:00
{
Name : "containerRegistryUser" ,
ResourceRef : [ ] config . ResourceReference {
{
Name : "commonPipelineEnvironment" ,
Param : "container/repositoryUsername" ,
} ,
} ,
Scope : [ ] string { "GENERAL" , "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { { Name : "dockerRegistryUser" } } ,
Default : os . Getenv ( "PIPER_containerRegistryUser" ) ,
} ,
{
Name : "containerRegistryPassword" ,
ResourceRef : [ ] config . ResourceReference {
{
Name : "commonPipelineEnvironment" ,
Param : "container/repositoryPassword" ,
} ,
} ,
Scope : [ ] string { "GENERAL" , "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { { Name : "dockerRegistryPassword" } } ,
Default : os . Getenv ( "PIPER_containerRegistryPassword" ) ,
} ,
2020-07-10 08:07:59 +02:00
{
Name : "customTlsCertificateLinks" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "[]string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : [ ] string { } ,
2020-07-10 08:07:59 +02:00
} ,
{
2020-09-16 14:50:09 +02:00
Name : "dockerConfigJSON" ,
ResourceRef : [ ] config . ResourceReference {
{
Name : "dockerConfigJsonCredentialsId" ,
Type : "secret" ,
} ,
2020-11-06 19:06:19 +02:00
{
2021-09-21 13:06:32 +02:00
Name : "dockerConfigFileVaultSecretName" ,
Type : "vaultSecretFile" ,
Default : "docker-config" ,
2020-11-06 19:06:19 +02:00
} ,
2020-09-16 14:50:09 +02:00
} ,
2021-09-10 16:10:32 +02:00
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
2020-09-16 14:50:09 +02:00
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2021-06-16 08:43:30 +02:00
Default : os . Getenv ( "PIPER_dockerConfigJSON" ) ,
2020-07-10 08:07:59 +02:00
} ,
{
Name : "dockerfilePath" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STAGES" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { { Name : "dockerfile" } } ,
2021-06-16 08:43:30 +02:00
Default : ` Dockerfile ` ,
2020-07-10 08:07:59 +02:00
} ,
2022-02-09 11:17:34 +02:00
{
Name : "targetArchitectures" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "GENERAL" , "STEPS" , "STAGES" , "PARAMETERS" } ,
Type : "[]string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
Default : [ ] string { ` ` } ,
} ,
2022-02-24 18:38:47 +02:00
{
Name : "readImageDigest" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "STEPS" , "STAGES" , "PARAMETERS" } ,
2022-11-07 15:27:05 +02:00
Type : "bool" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
Default : false ,
} ,
{
Name : "createBOM" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "GENERAL" , "STEPS" , "STAGES" , "PARAMETERS" } ,
2022-02-24 18:38:47 +02:00
Type : "bool" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
Default : false ,
} ,
2022-11-28 11:21:06 +02:00
{
Name : "syftDownloadUrl" ,
ResourceRef : [ ] config . ResourceReference { } ,
Scope : [ ] string { "PARAMETERS" , "STEPS" } ,
Type : "string" ,
Mandatory : false ,
Aliases : [ ] config . Alias { } ,
2022-12-13 11:51:14 +02:00
Default : ` https://github.com/anchore/syft/releases/download/v0.62.3/syft_0.62.3_linux_amd64.tar.gz ` ,
2022-11-28 11:21:06 +02:00
} ,
2020-07-10 08:07:59 +02:00
} ,
} ,
2020-11-20 09:13:59 +02:00
Containers : [ ] config . Container {
2022-02-23 12:41:26 +02:00
{ Image : "gcr.io/kaniko-project/executor:debug" , EnvVars : [ ] config . EnvVar { { Name : "container" , Value : "docker" } } , Options : [ ] config . Option { { Name : "-u" , Value : "0" } , { Name : "--entrypoint" , Value : "" } } } ,
2020-11-20 09:13:59 +02:00
} ,
Outputs : config . StepOutputs {
Resources : [ ] config . StepResources {
{
Name : "commonPipelineEnvironment" ,
Type : "piperEnvironment" ,
Parameters : [ ] map [ string ] interface { } {
2021-12-15 10:40:50 +02:00
{ "name" : "container/registryUrl" } ,
{ "name" : "container/imageNameTag" } ,
2022-02-23 12:41:26 +02:00
{ "name" : "container/imageDigest" } ,
2022-02-07 08:58:41 +02:00
{ "name" : "container/imageNames" , "type" : "[]string" } ,
{ "name" : "container/imageNameTags" , "type" : "[]string" } ,
2022-02-23 12:41:26 +02:00
{ "name" : "container/imageDigests" , "type" : "[]string" } ,
2021-12-15 10:40:50 +02:00
{ "name" : "custom/buildSettingsInfo" } ,
2020-11-20 09:13:59 +02:00
} ,
} ,
2022-12-13 11:51:14 +02:00
{
Name : "reports" ,
Type : "reports" ,
Parameters : [ ] map [ string ] interface { } {
{ "filePattern" : "**/bom-*.xml" , "type" : "sbom" } ,
} ,
} ,
2020-11-20 09:13:59 +02:00
} ,
} ,
2020-07-10 08:07:59 +02:00
} ,
}
return theMetaData
}