mirror of
https://github.com/SAP/jenkins-library.git
synced 2024-12-16 11:09:33 +02:00
42b92d1bfe
* Adds GetLog() function to orchestrator * Fixes BUILD_NUMBER env variable * Fixes correct env var for JENKINS_HOME * Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation * Adds possibility to read errorJsons; updates splunk package for log files (WIP) * Uncommenting dev code * Adds GetLog() function to orchestrator * Fixes BUILD_NUMBER env variable * Fixes correct env var for JENKINS_HOME * Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation * Adds possibility to read errorJsons; updates splunk package for log files (WIP) * Uncommenting dev code * Adds GetRequest function which holds the response in memory (not saved to disk) * Implements GetLog() function for ADO, adds function to read PipelineRuntime * PAT has been revoked * Changes http package, s.t. if password only is required basic auth works too * Adds env variable for azure token, error handling in case of unauthenticated/nil response * Adds logging output in case env variable can not be read and fallback variable needs to be used * Adds usage of environment variables for auth, uses jenkins api * Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function * Adds initaliziation function for orchestrator authetnication * Adds settings struct for orchestrator authentication * Adds function to whole logfile to Splunk * Struct for pipeline related telemetry information * Increase messagebatch size to 10k * Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk * Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation * Changes content type to txt * Send telemetry independent of logfiles, increases amount of messages per file * Adds JobURL for orchestrators and UnknownOrchestrator as fallback * telemetry makes use of orchestrator specific information * Adds orchestrator independent correlationID * Adds custom fields for pipeline status * go fmt * Removes env var test - no env variables are read anymore * Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized * Removes Custom fields from telemetry as these can not be reflected in SWA * Adds custom telemetry information (piperHash,..) to each step telemetry information * Removes falltrough in case no orchestrator has been found * Updates tests for orchestrator package * Adds orchestrator import in generated files * Updates generator files for internal library * Adds orchestrator telemetry information to steps * Updates generated files, fatalHook writes to cpe * Go generate from master, go fmt * Adds Custom Data field LastErrorCode * Removes GetLog() test * Update init_unix.go * Update docker_integration_test_executor.go * Update integration_api_cli_test.go * Reverts go1.17 fmt formatting * Reverts go1.17 fmt formatting * Reverts go1.17 fmt formatting * Renames customTelemetryData to stepTelemetryData * Adjustments to orchestrator-package, cleanup, adds JobName * Adjusts commonPipelineEnvironment path * Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package * Go fmt * Changes path for errorDetails, adds debug information * Removes custom fields from step, adds orchestrator, commithash to baseMetadata * Adjusts tests for telemetry package * Adds tests for orchestrator * Updates generated files, initalization of splunk client only if its available in the config * Fixes typo in helper go * Update pkg/http/downloader.go * Update pkg/http/downloader.go * Update pkg/log/fatalHook.go * Update fatalHook.go * Update pkg/splunk/splunk.go * Update pkg/telemetry/data.go * Adds GetBuildStatus() and GetAPIInformation() to orchestrators * error formatting * Bugfix: dont send telemetry data if disabled, adjusts test * go fmt * Use correct error handling * Update pkg/telemetry/telemetry.go * Fixes telemetry disabled in the tests * Fixes http tests * Log fatal errors to logFile * Adds CustomReportingConfig to hooks * Cleanup comments in splunk package * Adds possibility to send telemetry to custom endpoint * Adds debug output for the payload * Debug output for the payload as a string * Adds test cases for changes in telemetry package * go fmt * Adds generated files for new step * Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17 * Adjusts test for sonarExecuteScan * Adjusts test for sonarExecuteScan * Adds explanation for customreportingConfig * Makes disableing of customSend more obvious * Adds custom step reporting to each step, updates generated files, adjusts helper testdata * fixes unit test wrong usage of logging * Send pipeline data altough there has been no error, adjust test cases * Reverts changes for customReporting * Updates generated files, removes customReporting * Removes writing errorDetails to CPE * Reverts usage of customreporting * go fmt * reverts changes in http_test * reverts changes in http_test * Skips integration cnb test Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
566 lines
14 KiB
Go
566 lines
14 KiB
Go
package splunk
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"os"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
piperhttp "github.com/SAP/jenkins-library/pkg/http"
|
|
"github.com/SAP/jenkins-library/pkg/log"
|
|
"github.com/SAP/jenkins-library/pkg/telemetry"
|
|
"github.com/jarcoal/httpmock"
|
|
)
|
|
|
|
func TestInitialize(t *testing.T) {
|
|
type args struct {
|
|
correlationID string
|
|
dsn string
|
|
token string
|
|
index string
|
|
sendLogs bool
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
args args
|
|
wantErr bool
|
|
}{
|
|
{"Testing initialize splunk",
|
|
args{
|
|
correlationID: "correlationID",
|
|
dsn: "https://splunkURL.sap/services/collector",
|
|
token: "SECRET-TOKEN",
|
|
index: "test-index",
|
|
sendLogs: false,
|
|
},
|
|
false,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
splunkClient := &Splunk{}
|
|
if err := splunkClient.Initialize(tt.args.correlationID, tt.args.dsn, tt.args.token, tt.args.index, tt.args.sendLogs); (err != nil) != tt.wantErr {
|
|
t.Errorf("Initialize() error = %v, wantErr %v", err, tt.wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSend(t *testing.T) {
|
|
|
|
type args struct {
|
|
telemetryData *telemetry.Data
|
|
logCollector *log.CollectorHook
|
|
sendLogs bool
|
|
maxBatchSize int
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
args args
|
|
wantErr bool
|
|
payloadLength int
|
|
logLength int // length of log per payload
|
|
}{
|
|
{name: "Testing Success Step - Send Telemetry Only",
|
|
args: args{
|
|
telemetryData: &telemetry.Data{
|
|
BaseData: telemetry.BaseData{},
|
|
BaseMetaData: telemetry.BaseMetaData{},
|
|
CustomData: telemetry.CustomData{
|
|
Duration: "100",
|
|
ErrorCode: "0",
|
|
ErrorCategory: "DEBUG",
|
|
},
|
|
},
|
|
logCollector: &log.CollectorHook{CorrelationID: "DEBUG",
|
|
Messages: []log.Message{
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 0",
|
|
},
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 1",
|
|
},
|
|
}},
|
|
sendLogs: false,
|
|
},
|
|
wantErr: false,
|
|
payloadLength: 1,
|
|
logLength: 0,
|
|
},
|
|
{name: "Testing Success Step - Send Telemetry Only Although sendLogs Active",
|
|
args: args{
|
|
telemetryData: &telemetry.Data{
|
|
BaseData: telemetry.BaseData{},
|
|
BaseMetaData: telemetry.BaseMetaData{},
|
|
CustomData: telemetry.CustomData{
|
|
Duration: "100",
|
|
ErrorCode: "0",
|
|
ErrorCategory: "DEBUG",
|
|
},
|
|
},
|
|
logCollector: &log.CollectorHook{CorrelationID: "DEBUG",
|
|
Messages: []log.Message{
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 0",
|
|
},
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 1",
|
|
},
|
|
}},
|
|
sendLogs: true,
|
|
},
|
|
wantErr: false,
|
|
payloadLength: 1,
|
|
logLength: 0,
|
|
},
|
|
{name: "Testing Failure Step - Send Telemetry Only",
|
|
args: args{
|
|
telemetryData: &telemetry.Data{
|
|
BaseData: telemetry.BaseData{},
|
|
BaseMetaData: telemetry.BaseMetaData{},
|
|
CustomData: telemetry.CustomData{
|
|
Duration: "100",
|
|
ErrorCode: "0",
|
|
ErrorCategory: "DEBUG",
|
|
},
|
|
},
|
|
logCollector: &log.CollectorHook{CorrelationID: "DEBUG",
|
|
Messages: []log.Message{
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 0",
|
|
},
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 1",
|
|
},
|
|
}},
|
|
sendLogs: false,
|
|
maxBatchSize: 1000,
|
|
},
|
|
wantErr: false,
|
|
payloadLength: 1,
|
|
logLength: 0,
|
|
},
|
|
{name: "Testing Failure Step - Send Telemetry and Logs",
|
|
args: args{
|
|
telemetryData: &telemetry.Data{
|
|
BaseData: telemetry.BaseData{},
|
|
BaseMetaData: telemetry.BaseMetaData{},
|
|
CustomData: telemetry.CustomData{
|
|
Duration: "100",
|
|
ErrorCode: "1",
|
|
ErrorCategory: "DEBUG",
|
|
},
|
|
},
|
|
logCollector: &log.CollectorHook{CorrelationID: "DEBUG",
|
|
Messages: []log.Message{
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 0",
|
|
},
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 1",
|
|
},
|
|
}},
|
|
sendLogs: true,
|
|
maxBatchSize: 1000,
|
|
},
|
|
wantErr: false,
|
|
payloadLength: 1,
|
|
logLength: 2,
|
|
},
|
|
{name: "Testing len(maxBatchSize)==len(logMessages)",
|
|
args: args{
|
|
telemetryData: &telemetry.Data{
|
|
BaseData: telemetry.BaseData{},
|
|
BaseMetaData: telemetry.BaseMetaData{},
|
|
CustomData: telemetry.CustomData{
|
|
Duration: "100",
|
|
ErrorCode: "1",
|
|
},
|
|
},
|
|
logCollector: &log.CollectorHook{CorrelationID: "DEBUG",
|
|
Messages: []log.Message{
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 0",
|
|
},
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 1",
|
|
},
|
|
}},
|
|
sendLogs: true,
|
|
maxBatchSize: 2,
|
|
},
|
|
wantErr: false,
|
|
payloadLength: 1,
|
|
logLength: 2,
|
|
},
|
|
{name: "Testing len(maxBatchSize)<len(logMessages)",
|
|
args: args{
|
|
telemetryData: &telemetry.Data{
|
|
BaseData: telemetry.BaseData{},
|
|
BaseMetaData: telemetry.BaseMetaData{},
|
|
CustomData: telemetry.CustomData{
|
|
Duration: "100",
|
|
ErrorCode: "1",
|
|
},
|
|
},
|
|
logCollector: &log.CollectorHook{CorrelationID: "DEBUG",
|
|
Messages: []log.Message{
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 0",
|
|
},
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 1",
|
|
},
|
|
}},
|
|
sendLogs: true,
|
|
maxBatchSize: 1,
|
|
},
|
|
wantErr: false,
|
|
payloadLength: 2,
|
|
logLength: 1, // equal to maxBatchSize
|
|
},
|
|
{name: "Testing len(maxBatchSize)>len(logMessages)",
|
|
args: args{
|
|
telemetryData: &telemetry.Data{
|
|
BaseData: telemetry.BaseData{},
|
|
BaseMetaData: telemetry.BaseMetaData{},
|
|
CustomData: telemetry.CustomData{},
|
|
},
|
|
logCollector: &log.CollectorHook{CorrelationID: "DEBUG",
|
|
Messages: []log.Message{
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 0",
|
|
},
|
|
{
|
|
Time: time.Time{},
|
|
Level: 0,
|
|
Message: "DEBUG",
|
|
Data: "DEBUG 1",
|
|
},
|
|
}},
|
|
sendLogs: true,
|
|
maxBatchSize: 1000,
|
|
},
|
|
wantErr: false,
|
|
payloadLength: 1,
|
|
logLength: 2,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
httpmock.Activate()
|
|
defer httpmock.DeactivateAndReset()
|
|
|
|
fakeUrl := "https://splunk.example.com/services/collector"
|
|
// Our database of received payloads
|
|
var payloads []Details
|
|
httpmock.RegisterResponder("POST", fakeUrl,
|
|
func(req *http.Request) (*http.Response, error) {
|
|
splunkMessage := Details{}
|
|
if err := json.NewDecoder(req.Body).Decode(&splunkMessage); err != nil {
|
|
return httpmock.NewStringResponse(400, ""), nil
|
|
}
|
|
|
|
defer req.Body.Close()
|
|
payloads = append(payloads, splunkMessage)
|
|
|
|
resp, err := httpmock.NewJsonResponse(200, splunkMessage)
|
|
if err != nil {
|
|
return httpmock.NewStringResponse(500, ""), nil
|
|
}
|
|
return resp, nil
|
|
},
|
|
)
|
|
|
|
client := piperhttp.Client{}
|
|
client.SetOptions(piperhttp.ClientOptions{
|
|
MaxRequestDuration: 5 * time.Second,
|
|
Token: "TOKEN",
|
|
TransportSkipVerification: true,
|
|
UseDefaultTransport: true,
|
|
MaxRetries: -1,
|
|
})
|
|
|
|
splunkClient := &Splunk{
|
|
splunkClient: client,
|
|
splunkDsn: fakeUrl,
|
|
splunkIndex: "index",
|
|
correlationID: "DEBUG",
|
|
postMessagesBatchSize: tt.args.maxBatchSize,
|
|
sendLogs: tt.args.sendLogs,
|
|
}
|
|
if err := splunkClient.Send(*tt.args.telemetryData, tt.args.logCollector); (err != nil) != tt.wantErr {
|
|
t.Errorf("Send() error = %v, wantErr %v", err, tt.wantErr)
|
|
}
|
|
if len(payloads) != tt.payloadLength {
|
|
t.Errorf("Send() error, wanted %v payloads, got %v.", tt.payloadLength, len(payloads))
|
|
}
|
|
|
|
// The case if more than one payload is present is covered in the if statement above.
|
|
if len(payloads[0].Event.Messages) != tt.logLength {
|
|
t.Errorf("Send() error, wanted %v event messages, got %v.", tt.logLength, len(payloads[0].Event.Messages))
|
|
}
|
|
splunkClient = nil
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_prepareTelemetry(t *testing.T) {
|
|
type args struct {
|
|
telemetryData telemetry.Data
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
args args
|
|
want MonitoringData
|
|
}{
|
|
{name: "Testing prepare telemetry information",
|
|
args: args{
|
|
telemetryData: telemetry.Data{
|
|
BaseData: telemetry.BaseData{},
|
|
BaseMetaData: telemetry.BaseMetaData{},
|
|
CustomData: telemetry.CustomData{
|
|
Duration: "1234",
|
|
ErrorCode: "0",
|
|
ErrorCategory: "Undefined",
|
|
},
|
|
},
|
|
},
|
|
want: MonitoringData{
|
|
PipelineUrlHash: "",
|
|
BuildUrlHash: "",
|
|
StageName: "",
|
|
StepName: "",
|
|
ExitCode: "0",
|
|
Duration: "1234",
|
|
ErrorCode: "0",
|
|
ErrorCategory: "Undefined",
|
|
CorrelationID: "Correlation-Test",
|
|
CommitHash: "N/A",
|
|
Branch: "N/A",
|
|
GitOwner: "N/A",
|
|
GitRepository: "N/A",
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
splunkClient := &Splunk{}
|
|
err := splunkClient.Initialize("Correlation-Test", "splunkUrl", "TOKEN", "index", false)
|
|
if err != nil {
|
|
t.Errorf("Error Initalizing Splunk. %v", err)
|
|
}
|
|
if got := splunkClient.prepareTelemetry(tt.args.telemetryData); !reflect.DeepEqual(got, tt.want) {
|
|
t.Errorf("prepareTelemetry() = %v, want %v", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_tryPostMessages(t *testing.T) {
|
|
type args struct {
|
|
telemetryData MonitoringData
|
|
messages []log.Message
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
args args
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "Test HTTP Success",
|
|
args: args{
|
|
telemetryData: MonitoringData{
|
|
PipelineUrlHash: "1234",
|
|
BuildUrlHash: "5678",
|
|
StageName: "deploy",
|
|
StepName: "cloudFoundryDeploy",
|
|
ExitCode: "0",
|
|
Duration: "12345678",
|
|
ErrorCode: "0",
|
|
ErrorCategory: "undefined",
|
|
CorrelationID: "123",
|
|
CommitHash: "a6bc",
|
|
Branch: "prod",
|
|
GitOwner: "N/A",
|
|
GitRepository: "N/A",
|
|
},
|
|
messages: []log.Message{},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "Test HTTP Failure",
|
|
args: args{
|
|
telemetryData: MonitoringData{
|
|
PipelineUrlHash: "1234",
|
|
BuildUrlHash: "5678",
|
|
StageName: "deploy",
|
|
StepName: "cloudFoundryDeploy",
|
|
ExitCode: "0",
|
|
Duration: "12345678",
|
|
ErrorCode: "0",
|
|
ErrorCategory: "undefined",
|
|
CorrelationID: "123",
|
|
CommitHash: "a6bc",
|
|
Branch: "prod",
|
|
GitOwner: "N/A",
|
|
GitRepository: "N/A",
|
|
},
|
|
messages: []log.Message{},
|
|
},
|
|
wantErr: true,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
httpmock.Activate()
|
|
fakeUrl := "https://splunk.example.com/services/collector"
|
|
defer httpmock.DeactivateAndReset()
|
|
httpmock.RegisterResponder("POST", fakeUrl,
|
|
func(req *http.Request) (*http.Response, error) {
|
|
if tt.wantErr == true {
|
|
return &http.Response{
|
|
Status: "400",
|
|
StatusCode: 400,
|
|
Proto: "",
|
|
ProtoMajor: 0,
|
|
ProtoMinor: 0,
|
|
Header: nil,
|
|
Body: nil,
|
|
ContentLength: 0,
|
|
TransferEncoding: nil,
|
|
Close: false,
|
|
Uncompressed: false,
|
|
Trailer: nil,
|
|
Request: req,
|
|
TLS: nil,
|
|
}, nil
|
|
}
|
|
return httpmock.NewStringResponse(200, ""), nil
|
|
},
|
|
)
|
|
client := piperhttp.Client{}
|
|
client.SetOptions(piperhttp.ClientOptions{
|
|
MaxRequestDuration: 5 * time.Second,
|
|
Token: "TOKEN",
|
|
TransportSkipVerification: true,
|
|
UseDefaultTransport: true,
|
|
MaxRetries: -1,
|
|
})
|
|
splunkClient := &Splunk{
|
|
splunkClient: client,
|
|
splunkDsn: fakeUrl,
|
|
splunkIndex: "index",
|
|
correlationID: "DEBUG",
|
|
postMessagesBatchSize: 1000,
|
|
}
|
|
if err := splunkClient.tryPostMessages(tt.args.telemetryData, tt.args.messages); (err != nil) != tt.wantErr {
|
|
t.Errorf("tryPostMessages() error = %v, wantErr %v", err, tt.wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_readPipelineEnvironment(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
result string
|
|
createFile bool
|
|
}{
|
|
{
|
|
name: "Test read pipelineEnvironment files not available",
|
|
result: "N/A",
|
|
createFile: false,
|
|
},
|
|
{
|
|
name: "Test read pipelineEnvironment files available",
|
|
result: "master",
|
|
createFile: true,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
|
|
if tt.createFile {
|
|
|
|
// creating temporarily folders
|
|
path := ".pipeline/commonPipelineEnvironment/"
|
|
err := os.MkdirAll(path, os.ModePerm)
|
|
if err != nil {
|
|
t.Errorf("Could not create .pipeline/ folders: %v", err)
|
|
}
|
|
|
|
err = os.Mkdir(path+"git/", os.ModePerm)
|
|
if err != nil {
|
|
t.Errorf("Could not create git folder: %v", err)
|
|
}
|
|
|
|
// creating temporarily files with dummy content
|
|
branch := []byte("master")
|
|
err = ioutil.WriteFile(path+"git/branch", branch, 0644)
|
|
if err != nil {
|
|
t.Errorf("Could not create branch file: %v", err)
|
|
}
|
|
|
|
}
|
|
result := readCommonPipelineEnvironment("git/branch")
|
|
if result != tt.result {
|
|
t.Errorf("readCommonPipelineEnvironment() got = %v, want %v", result, tt.result)
|
|
}
|
|
|
|
if tt.createFile {
|
|
// deletes temp files
|
|
err := os.RemoveAll(".pipeline")
|
|
if err != nil {
|
|
t.Errorf("Could not delete .pipeline folder: %v", err)
|
|
}
|
|
|
|
}
|
|
})
|
|
}
|
|
}
|