diff --git a/generator/go.mod b/generator/go.mod new file mode 100644 index 000000000..868c5904f --- /dev/null +++ b/generator/go.mod @@ -0,0 +1,69 @@ +module github.com/SAP/jenkins-library/generator + +go 1.24.0 + +toolchain go1.24.1 + +replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.16 + +require ( + github.com/Masterminds/sprig v2.22.0+incompatible + github.com/SAP/jenkins-library v1.449.0 + github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 + github.com/stretchr/testify v1.10.0 +) + +require ( + github.com/Jeffail/gabs/v2 v2.7.0 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/bmatcuk/doublestar v1.3.4 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/getsentry/sentry-go v0.31.1 // indirect + github.com/go-jose/go-jose/v4 v4.0.1 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.14.1 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/go-github/v68 v68.0.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/hcl v1.0.1-vault-5 // indirect + github.com/hashicorp/vault/api v1.15.0 // indirect + github.com/hashicorp/vault/api/auth/approle v0.8.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect + github.com/imdario/mergo v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/leodido/go-urn v1.2.4 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/motemen/go-nuts v0.0.0-20220604134737-2658d0104f31 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/cobra v1.9.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.37.0 // indirect + golang.org/x/oauth2 v0.28.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.11.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + mvdan.cc/xurls/v2 v2.4.0 // indirect +) diff --git a/generator/helper/goUtils.go b/generator/helper/goUtils.go new file mode 100644 index 000000000..4d8d0a65d --- /dev/null +++ b/generator/helper/goUtils.go @@ -0,0 +1,72 @@ +package helper + +import ( + "fmt" + "io" + "os" + + "github.com/ghodss/yaml" +) + +// StepHelperData is used to transport the needed parameters and functions from the step generator to the step generation. +type StepHelperData struct { + OpenFile func(s string) (io.ReadCloser, error) + WriteFile func(filename string, data []byte, perm os.FileMode) error + ExportPrefix string +} + +// ContextDefaultData holds the meta data and the default data for the context default parameter descriptions +type ContextDefaultData struct { + Metadata ContextDefaultMetadata `json:"metadata"` + Parameters []ContextDefaultParameters `json:"params"` +} + +// ContextDefaultMetadata holds meta data for the context default parameter descripten (name, description, long description) +type ContextDefaultMetadata struct { + Name string `json:"name"` + Description string `json:"description"` + LongDescription string `json:"longDescription,omitempty"` +} + +// ContextDefaultParameters holds the description for the context default parameters +type ContextDefaultParameters struct { + Name string `json:"name"` + Description string `json:"description"` + Scope []string `json:"scope"` +} + +// ReadPipelineContextDefaultData loads step definition in yaml format +func (c *ContextDefaultData) readPipelineContextDefaultData(metadata io.ReadCloser) { + defer metadata.Close() + content, err := io.ReadAll(metadata) + checkError(err) + err = yaml.Unmarshal(content, &c) + checkError(err) +} + +// ReadContextDefaultMap maps the default descriptions into a map +func (c *ContextDefaultData) readContextDefaultMap() map[string]interface{} { + var m map[string]interface{} = make(map[string]interface{}) + + for _, param := range c.Parameters { + m[param.Name] = param + } + + return m +} + +func checkError(err error) { + if err != nil { + fmt.Printf("Error occurred: %v\n", err) + os.Exit(1) + } +} + +func contains(v []string, s string) bool { + for _, i := range v { + if i == s { + return true + } + } + return false +} diff --git a/generator/helper/helper.go b/generator/helper/helper.go new file mode 100644 index 000000000..a1b6fbdec --- /dev/null +++ b/generator/helper/helper.go @@ -0,0 +1,751 @@ +package helper + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "text/template" + + "github.com/Masterminds/sprig" + "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/piperutils" +) + +type stepInfo struct { + CobraCmdFuncName string + CreateCmdVar string + ExportPrefix string + FlagsFunc string + Long string + StepParameters []config.StepParameters + StepAliases []config.Alias + OSImport bool + OutputResources []map[string]string + Short string + StepFunc string + StepName string + StepSecrets []string + Containers []config.Container + Sidecars []config.Container + Outputs config.StepOutputs + Resources []config.StepResources + Secrets []config.StepSecrets +} + +// StepGoTemplate ... +const stepGoTemplate = `// Code generated by piper's step-generator. DO NOT EDIT. + +package cmd + +{{ $reportsOutputExists := false -}} +{{ $influxOutputExists := false -}} +{{ $piperEnvironmentOutputExists := false -}} +{{ if .OutputResources -}} + {{ range $notused, $oRes := .OutputResources -}} + {{ if eq (index $oRes "type") "reports" -}}{{ $reportsOutputExists = true -}}{{ end -}} + {{ if eq (index $oRes "type") "influx" -}}{{ $influxOutputExists = true -}}{{ end -}} + {{ if eq (index $oRes "type") "piperEnvironment" -}}{{ $piperEnvironmentOutputExists = true -}}{{ end -}} + {{ end -}} +{{ end -}} + +import ( + "fmt" + "path/filepath" + + "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/piperenv" + "github.com/SAP/jenkins-library/pkg/telemetry" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + StepName = {{ .StepName | quote }} +) + +type generalConfig struct { + Verbose bool +} + +type {{ .StepName }}Options struct { + generalConfig + {{- $names := list ""}} + {{- range $key, $value := uniqueName .StepParameters }} + {{ if ne (has $value.Name $names) true -}} + {{ $names | last }}{{ $value.Name | golangName }} {{ $value.Type }} ` + "`json:\"{{$value.Name}},omitempty\"" + + "{{ if or $value.PossibleValues $value.MandatoryIf}} validate:\"" + + "{{ if $value.PossibleValues }}possible-values={{ range $i,$a := $value.PossibleValues }}{{if gt $i 0 }} {{ end }}{{.}}{{ end }}{{ end }}" + + "{{ if and $value.PossibleValues $value.MandatoryIf }},{{ end }}" + + "{{ if $value.MandatoryIf }}required_if={{ range $i,$a := $value.MandatoryIf }}{{ if gt $i 0 }} {{ end }}{{ $a.Name | title }} {{ $a.Value }}{{ end }}{{ end }}" + + "\"{{ end }}`" + ` + {{- else -}} + {{- $names = append $names $value.Name }} {{ end -}} + {{ end }} +} + +func (cfg *{{ .StepName }}Options) readInValues() error { + err := viper.Unmarshal(cfg) + if err != nil { + return fmt.Errorf("unable to read inputs step configuration, %v", err) + } + + return nil +} + +// {{.CobraCmdFuncName}} {{.Short}} +func {{.CobraCmdFuncName}}() *cobra.Command { + var stepConfig {{.StepName}}Options + var commonPipelineEnvironment golangBuildCommonPipelineEnvironment + var dummyTelemetryData telemetry.CustomData + + var {{.StepName}}Cmd = &cobra.Command{ + Use: StepName, + Short: {{.Short | quote }}, + Long: {{ $tick := "` + "`" + `" }}{{ $tick }}{{.Long | longName }}{{ $tick }}, + PreRunE: func(cmd *cobra.Command, _ []string) error { + err := stepConfig.readInValues() + if err != nil { + return err + } + + log.SetStepName(StepName) + log.SetVerbose(stepConfig.Verbose) + + {{- range $key, $value := .StepSecrets }} + log.RegisterSecret(stepConfig.{{ $value | golangName }}){{end}} + + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + {{.StepName}}(stepConfig, &dummyTelemetryData, &commonPipelineEnvironment) + log.Entry().Info("SUCCESS") + }, + } + + defineInputSources({{.StepName}}Cmd, &stepConfig) + return {{.StepName}}Cmd +} + +func defineInputSources(stepCmd *cobra.Command, stepConfig *golangBuildOptions) { + // General configuration + stepCmd.Flags().BoolVarP(&stepConfig.Verbose, "verbose", "v", false, "Enables verbose output for the step.") + + // Define flags + {{- range $key, $value := uniqueName .StepParameters }} + {{ if isCLIParam $value.Type }}stepCmd.Flags().{{ $value.Type | flagType }}(&stepConfig.{{ $value.Name | golangName }}, {{ $value.Name | quote }}, {{ $value.Default }}, {{ $value.Description | quote }}){{end}}{{ end }} + {{- printf "\n" }} + {{- range $key, $value := .StepParameters }} + {{- if $value.Mandatory }} + stepCmd.MarkFlagRequired({{ $value.Name | quote }}) + {{- end }} + {{- if $value.DeprecationMessage }} + stepCmd.Flags().MarkDeprecated({{ $value.Name | quote }}, {{ $value.DeprecationMessage | quote }}) + {{- end }} + {{- end }} + + bindEnvToFlag(stepCmd) + + // Set prefix for all environment variables. e.g. PIPER_STEP_verbose + viper.SetEnvPrefix("PIPER_STEP") + viper.AutomaticEnv() +} + +// Bind environment variables to flags +func bindEnvToFlag(stepCmd *cobra.Command) { + _ = viper.BindPFlag("verbose", stepCmd.Flags().Lookup("verbose")) + {{- printf "\n" }} + {{- range $key, $value := uniqueName .StepParameters }} + _ = viper.BindPFlag({{ $value.Name | quote }}, stepCmd.Flags().Lookup({{ $value.Name | quote }})) + {{- end }} +} + +{{ range $notused, $oRes := .OutputResources }} +{{ index $oRes "def"}} +{{ end }} + +{{ define "resourceRefs"}} + {{ "{" }} + Name: {{- .Name | quote }}, + {{- if .Param }} + Param: {{ .Param | quote }}, + {{- end }} + {{- if .Type }} + Type: {{ .Type | quote }}, + {{- if .Default }} + Default: {{ .Default | quote }}, + {{- end}} + {{- end }} + {{ "}" }}, + {{- nindent 24 ""}} +{{- end -}} +` + +// StepTestGoTemplate ... +const stepTestGoTemplate = `//go:build unit +// +build unit + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test{{.CobraCmdFuncName}}(t *testing.T) { + t.Parallel() + + testCmd := {{.CobraCmdFuncName}}() + + // only high level testing performed - details are tested in step generation procedure + assert.Equal(t, {{ .StepName | quote }}, testCmd.Use, "command name incorrect") + +} +` + +const stepGoImplementationTemplate = `package cmd +import ( + "fmt" + "github.com/SAP/jenkins-library/pkg/command" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/telemetry" + "github.com/SAP/jenkins-library/pkg/piperutils" +) + +type {{.StepName}}Utils interface { + command.ExecRunner + + FileExists(filename string) (bool, error) + + // Add more methods here, or embed additional interfaces, or remove/replace as required. + // The {{.StepName}}Utils interface should be descriptive of your runtime dependencies, + // i.e. include everything you need to be able to mock in tests. + // Unit tests shall be executable in parallel (not depend on global state), and don't (re-)test dependencies. +} + +type {{.StepName}}UtilsBundle struct { + *command.Command + *piperutils.Files + + // Embed more structs as necessary to implement methods or interfaces you add to {{.StepName}}Utils. + // Structs embedded in this way must each have a unique set of methods attached. + // If there is no struct which implements the method you need, attach the method to + // {{.StepName}}UtilsBundle and forward to the implementation of the dependency. +} + +func new{{.StepName | title}}Utils() {{.StepName}}Utils { + utils := {{.StepName}}UtilsBundle{ + Command: &command.Command{}, + Files: &piperutils.Files{}, + } + // Reroute command output to logging framework + utils.Stdout(log.Writer()) + utils.Stderr(log.Writer()) + return &utils +} + +func {{.StepName}}(config {{ .StepName }}Options, telemetryData *telemetry.CustomData{{ range $notused, $oRes := .OutputResources}}, {{ index $oRes "name" }} *{{ index $oRes "objectname" }}{{ end }}) { + // Utils can be used wherever the command.ExecRunner interface is expected. + // It can also be used for example as a mavenExecRunner. + utils := new{{.StepName | title}}Utils() + + // For HTTP calls import piperhttp "github.com/SAP/jenkins-library/pkg/http" + // and use a &piperhttp.Client{} in a custom system + // Example: step checkmarxExecuteScan.go + + // Error situations should be bubbled up until they reach the line below which will then stop execution + // through the log.Entry().Fatal() call leading to an os.Exit(1) in the end. + err := run{{.StepName | title}}(&config, telemetryData, utils{{ range $notused, $oRes := .OutputResources}}, {{ index $oRes "name" }}{{ end }}) + if err != nil { + log.Entry().WithError(err).Fatal("step execution failed") + } +} + +func run{{.StepName | title}}(config *{{ .StepName }}Options, telemetryData *telemetry.CustomData, utils {{.StepName}}Utils{{ range $notused, $oRes := .OutputResources}}, {{ index $oRes "name" }} *{{ index $oRes "objectname" }} {{ end }}) error { + log.Entry().WithField("LogField", "Log field content").Info("This is just a demo for a simple step.") + + // Example of calling methods from external dependencies directly on utils: + exists, err := utils.FileExists("file.txt") + if err != nil { + // It is good practice to set an error category. + // Most likely you want to do this at the place where enough context is known. + log.SetErrorCategory(log.ErrorConfiguration) + // Always wrap non-descriptive errors to enrich them with context for when they appear in the log: + return fmt.Errorf("failed to check for important file: %w", err) + } + if !exists { + log.SetErrorCategory(log.ErrorConfiguration) + return fmt.Errorf("cannot run without important file") + } + + return nil +} +` + +const stepGoImplementationTestTemplate = `package cmd + +import ( + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/stretchr/testify/assert" + "testing" +) + +type {{.StepName}}MockUtils struct { + *mock.ExecMockRunner + *mock.FilesMock +} + +func new{{.StepName | title}}TestsUtils() {{.StepName}}MockUtils { + utils := {{.StepName}}MockUtils{ + ExecMockRunner: &mock.ExecMockRunner{}, + FilesMock: &mock.FilesMock{}, + } + return utils +} + +func TestRun{{.StepName | title}}(t *testing.T) { + t.Parallel() + + t.Run("happy path", func(t *testing.T) { + t.Parallel() + // init + config := {{.StepName}}Options{} + + utils := new{{.StepName | title}}TestsUtils() + utils.AddFile("file.txt", []byte("dummy content")) + + // test + err := run{{.StepName | title}}(&config, nil, utils) + + // assert + assert.NoError(t, err) + }) + + t.Run("error path", func(t *testing.T) { + t.Parallel() + // init + config := {{.StepName}}Options{} + + utils := new{{.StepName | title}}TestsUtils() + + // test + err := run{{.StepName | title}}(&config, nil, utils) + + // assert + assert.EqualError(t, err, "cannot run without important file") + }) +} +` + +const metadataGeneratedFileName = "metadata_generated.go" +const metadataGeneratedTemplate = `// Code generated by piper's step-generator. DO NOT EDIT. + +package cmd + +import "github.com/SAP/jenkins-library/pkg/config" + +// GetStepMetadata return a map with all the step metadata mapped to their stepName +func GetAllStepMetadata() map[string]config.StepData { + return map[string]config.StepData{ + {{range $stepName := .Steps }} {{ $stepName | quote }}: {{$stepName}}Metadata(), + {{end}} + } +} +` + +// ProcessMetaFiles generates step coding based on step configuration provided in yaml files +func ProcessMetaFiles(metadataFiles []string, targetDir string, stepHelperData StepHelperData) error { + + allSteps := struct{ Steps []string }{} + for key := range metadataFiles { + + var stepData config.StepData + + configFilePath := metadataFiles[key] + + metadataFile, err := stepHelperData.OpenFile(configFilePath) + checkError(err) + defer metadataFile.Close() + + fmt.Printf("Reading file %v\n", configFilePath) + + err = stepData.ReadPipelineStepData(metadataFile) + checkError(err) + + stepName := stepData.Metadata.Name + fmt.Printf("Step name: %v\n", stepName) + if stepName+".yaml" != filepath.Base(configFilePath) { + fmt.Printf("Expected file %s to have name %s.yaml (.yaml)\n", configFilePath, filepath.Join(filepath.Dir(configFilePath), stepName)) + os.Exit(1) + } + allSteps.Steps = append(allSteps.Steps, stepName) + + for _, parameter := range stepData.Spec.Inputs.Parameters { + for _, mandatoryIfCase := range parameter.MandatoryIf { + if mandatoryIfCase.Name == "" || mandatoryIfCase.Value == "" { + return errors.New("invalid mandatoryIf option") + } + } + } + + osImport := false + osImport, err = setDefaultParameters(&stepData) + checkError(err) + + myStepInfo, err := getStepInfo(&stepData, osImport, stepHelperData.ExportPrefix) + checkError(err) + + step := stepTemplate(myStepInfo, "step", stepGoTemplate) + err = stepHelperData.WriteFile(filepath.Join(targetDir, fmt.Sprintf("%v_generated.go", stepName)), step, 0644) + checkError(err) + + test := stepTemplate(myStepInfo, "stepTest", stepTestGoTemplate) + err = stepHelperData.WriteFile(filepath.Join(targetDir, fmt.Sprintf("%v_generated_test.go", stepName)), test, 0644) + checkError(err) + + exists, _ := piperutils.FileExists(filepath.Join(targetDir, fmt.Sprintf("%v.go", stepName))) + if !exists { + impl := stepImplementation(myStepInfo, "impl", stepGoImplementationTemplate) + err = stepHelperData.WriteFile(filepath.Join(targetDir, fmt.Sprintf("%v.go", stepName)), impl, 0644) + checkError(err) + } + + exists, _ = piperutils.FileExists(filepath.Join(targetDir, fmt.Sprintf("%v_test.go", stepName))) + if !exists { + impl := stepImplementation(myStepInfo, "implTest", stepGoImplementationTestTemplate) + err = stepHelperData.WriteFile(filepath.Join(targetDir, fmt.Sprintf("%v_test.go", stepName)), impl, 0644) + checkError(err) + } + } + + // expose metadata functions + code := generateCode(allSteps, "metadata", metadataGeneratedTemplate, sprig.HermeticTxtFuncMap()) + err := stepHelperData.WriteFile(filepath.Join(targetDir, metadataGeneratedFileName), code, 0644) + checkError(err) + + return nil +} + +func setDefaultParameters(stepData *config.StepData) (bool, error) { + // ToDo: custom function for default handling, support all relevant parameter types + osImportRequired := false + for k, param := range stepData.Spec.Inputs.Parameters { + + if param.Default == nil { + switch param.Type { + case "bool": + // ToDo: Check if default should be read from env + param.Default = "false" + case "int": + param.Default = "0" + case "string": + param.Default = `""` + osImportRequired = true + case "[]string": + // ToDo: Check if default should be read from env + param.Default = "[]string{}" + case "map[string]interface{}", "[]map[string]interface{}": + // Currently we don't need to set a default here since in this case the default + // is never used. Needs to be changed in case we enable cli parameter handling + // for that type. + default: + return false, fmt.Errorf("Meta data type not set or not known: '%v'", param.Type) + } + } else { + switch param.Type { + case "bool": + boolVal := "false" + if param.Default.(bool) == true { + boolVal = "true" + } + param.Default = boolVal + case "int": + param.Default = fmt.Sprintf("%v", param.Default) + case "string": + param.Default = fmt.Sprintf(`"%v"`, param.Default) + case "[]string": + param.Default = fmt.Sprintf(`[]string{"%v"}`, strings.Join(getStringSliceFromInterface(param.Default), `", "`)) + case "map[string]interface{}", "[]map[string]interface{}": + // Currently we don't need to set a default here since in this case the default + // is never used. Needs to be changed in case we enable cli parameter handling + // for that type. + default: + return false, fmt.Errorf("Meta data type not set or not known: '%v'", param.Type) + } + } + + stepData.Spec.Inputs.Parameters[k] = param + } + return osImportRequired, nil +} + +func getStepInfo(stepData *config.StepData, osImport bool, exportPrefix string) (stepInfo, error) { + oRes, err := getOutputResourceDetails(stepData) + + return stepInfo{ + StepName: stepData.Metadata.Name, + CobraCmdFuncName: fmt.Sprintf("%vCommand", piperutils.Title(stepData.Metadata.Name)), + CreateCmdVar: fmt.Sprintf("create%vCmd", piperutils.Title(stepData.Metadata.Name)), + Short: stepData.Metadata.Description, + Long: stepData.Metadata.LongDescription, + StepParameters: stepData.Spec.Inputs.Parameters, + StepAliases: stepData.Metadata.Aliases, + FlagsFunc: fmt.Sprintf("add%vFlags", piperutils.Title(stepData.Metadata.Name)), + OSImport: osImport, + OutputResources: oRes, + ExportPrefix: exportPrefix, + StepSecrets: getSecretFields(stepData), + Containers: stepData.Spec.Containers, + Sidecars: stepData.Spec.Sidecars, + Outputs: stepData.Spec.Outputs, + Resources: stepData.Spec.Inputs.Resources, + Secrets: stepData.Spec.Inputs.Secrets, + }, + err +} + +func getSecretFields(stepData *config.StepData) []string { + var secretFields []string + + for _, parameter := range stepData.Spec.Inputs.Parameters { + if parameter.Secret { + secretFields = append(secretFields, parameter.Name) + } + } + return secretFields +} + +func getOutputResourceDetails(stepData *config.StepData) ([]map[string]string, error) { + outputResources := []map[string]string{} + + for _, res := range stepData.Spec.Outputs.Resources { + currentResource := map[string]string{} + currentResource["name"] = res.Name + currentResource["type"] = res.Type + + switch res.Type { + case "piperEnvironment": + var envResource PiperEnvironmentResource + envResource.Name = res.Name + envResource.StepName = stepData.Metadata.Name + for _, param := range res.Parameters { + paramSections := strings.Split(fmt.Sprintf("%v", param["name"]), "/") + category := "" + name := paramSections[0] + if len(paramSections) > 1 { + name = strings.Join(paramSections[1:], "_") + category = paramSections[0] + if !contains(envResource.Categories, category) { + envResource.Categories = append(envResource.Categories, category) + } + } + envParam := PiperEnvironmentParameter{Category: category, Name: name, Type: fmt.Sprint(param["type"])} + envResource.Parameters = append(envResource.Parameters, envParam) + } + def, err := envResource.StructString() + if err != nil { + return outputResources, err + } + currentResource["def"] = def + currentResource["objectname"] = envResource.StructName() + outputResources = append(outputResources, currentResource) + case "influx": + var influxResource InfluxResource + influxResource.Name = res.Name + influxResource.StepName = stepData.Metadata.Name + for _, measurement := range res.Parameters { + influxMeasurement := InfluxMeasurement{Name: fmt.Sprintf("%v", measurement["name"])} + if fields, ok := measurement["fields"].([]interface{}); ok { + for _, field := range fields { + if fieldParams, ok := field.(map[string]interface{}); ok { + influxMeasurement.Fields = append(influxMeasurement.Fields, InfluxMetric{Name: fmt.Sprintf("%v", fieldParams["name"]), Type: fmt.Sprintf("%v", fieldParams["type"])}) + } + } + } + + if tags, ok := measurement["tags"].([]interface{}); ok { + for _, tag := range tags { + if tagParams, ok := tag.(map[string]interface{}); ok { + influxMeasurement.Tags = append(influxMeasurement.Tags, InfluxMetric{Name: fmt.Sprintf("%v", tagParams["name"])}) + } + } + } + influxResource.Measurements = append(influxResource.Measurements, influxMeasurement) + } + def, err := influxResource.StructString() + if err != nil { + return outputResources, err + } + currentResource["def"] = def + currentResource["objectname"] = influxResource.StructName() + outputResources = append(outputResources, currentResource) + case "reports": + var reportsResource ReportsResource + reportsResource.Name = res.Name + reportsResource.StepName = stepData.Metadata.Name + for _, param := range res.Parameters { + filePattern, _ := param["filePattern"].(string) + paramRef, _ := param["paramRef"].(string) + if filePattern == "" && paramRef == "" { + return outputResources, errors.New("both filePattern and paramRef cannot be empty at the same time") + } + stepResultType, _ := param["type"].(string) + reportsParam := ReportsParameter{FilePattern: filePattern, ParamRef: paramRef, Type: stepResultType} + reportsResource.Parameters = append(reportsResource.Parameters, reportsParam) + } + def, err := reportsResource.StructString() + if err != nil { + return outputResources, err + } + currentResource["def"] = def + currentResource["objectname"] = reportsResource.StructName() + outputResources = append(outputResources, currentResource) + } + } + + return outputResources, nil +} + +// MetadataFiles provides a list of all step metadata files +func MetadataFiles(sourceDirectory string) ([]string, error) { + + var metadataFiles []string + + err := filepath.Walk(sourceDirectory, func(path string, info os.FileInfo, err error) error { + if filepath.Ext(path) == ".yaml" { + metadataFiles = append(metadataFiles, path) + } + return nil + }) + if err != nil { + return metadataFiles, nil + } + return metadataFiles, nil +} + +func isCLIParam(myType string) bool { + return myType != "map[string]interface{}" && myType != "[]map[string]interface{}" +} + +func stepTemplate(myStepInfo stepInfo, templateName, goTemplate string) []byte { + funcMap := sprig.HermeticTxtFuncMap() + funcMap["flagType"] = flagType + funcMap["golangName"] = GolangNameTitle + funcMap["title"] = piperutils.Title + funcMap["longName"] = longName + funcMap["uniqueName"] = mustUniqName + funcMap["isCLIParam"] = isCLIParam + + return generateCode(myStepInfo, templateName, goTemplate, funcMap) +} + +func stepImplementation(myStepInfo stepInfo, templateName, goTemplate string) []byte { + funcMap := sprig.HermeticTxtFuncMap() + funcMap["title"] = piperutils.Title + funcMap["uniqueName"] = mustUniqName + + return generateCode(myStepInfo, templateName, goTemplate, funcMap) +} + +func generateCode(dataObject interface{}, templateName, goTemplate string, funcMap template.FuncMap) []byte { + tmpl, err := template.New(templateName).Funcs(funcMap).Parse(goTemplate) + checkError(err) + + var generatedCode bytes.Buffer + err = tmpl.Execute(&generatedCode, dataObject) + checkError(err) + + return generatedCode.Bytes() +} + +func longName(long string) string { + l := strings.ReplaceAll(long, "`", "` + \"`\" + `") + l = strings.TrimSpace(l) + return l +} + +func resourceFieldType(fieldType string) string { + // TODO: clarify why fields are initialized with and tags are initialized with '' + if len(fieldType) == 0 || fieldType == "" { + return "string" + } + return fieldType +} + +func golangName(name string) string { + properName := strings.Replace(name, "Api", "API", -1) + properName = strings.Replace(properName, "api", "API", -1) + properName = strings.Replace(properName, "Url", "URL", -1) + properName = strings.Replace(properName, "Id", "ID", -1) + properName = strings.Replace(properName, "Json", "JSON", -1) + properName = strings.Replace(properName, "json", "JSON", -1) + properName = strings.Replace(properName, "Tls", "TLS", -1) + return properName +} + +// GolangNameTitle returns name in title case with abbriviations in capital (API, URL, ID, JSON, TLS) +func GolangNameTitle(name string) string { + return piperutils.Title(golangName(name)) +} + +func flagType(paramType string) string { + var theFlagType string + switch paramType { + case "bool": + theFlagType = "BoolVar" + case "int": + theFlagType = "IntVar" + case "string": + theFlagType = "StringVar" + case "[]string": + theFlagType = "StringSliceVar" + default: + fmt.Printf("Meta data type not set or not known: '%v'\n", paramType) + os.Exit(1) + } + return theFlagType +} + +func getStringSliceFromInterface(iSlice interface{}) []string { + s := []string{} + + t, ok := iSlice.([]interface{}) + if ok { + for _, v := range t { + s = append(s, fmt.Sprintf("%v", v)) + } + } else { + s = append(s, fmt.Sprintf("%v", iSlice)) + } + + return s +} + +func mustUniqName(list []config.StepParameters) ([]config.StepParameters, error) { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + names := []string{} + dest := []config.StepParameters{} + var item config.StepParameters + for i := 0; i < l; i++ { + item = l2.Index(i).Interface().(config.StepParameters) + if !slices.Contains(names, item.Name) { + names = append(names, item.Name) + dest = append(dest, item) + } + } + + return dest, nil + default: + return nil, fmt.Errorf("Cannot find uniq on type %s", tp) + } +} diff --git a/generator/helper/helper_test.go b/generator/helper/helper_test.go new file mode 100644 index 000000000..6c23e22bf --- /dev/null +++ b/generator/helper/helper_test.go @@ -0,0 +1,334 @@ +//go:build unit +// +build unit + +package helper + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/SAP/jenkins-library/pkg/config" + "github.com/stretchr/testify/assert" +) + +func configOpenFileMock(name string) (io.ReadCloser, error) { + meta1 := `metadata: + name: testStep + aliases: + - name: testStepAlias + deprecated: true + description: Test description + longDescription: | + Long Test description +spec: + outputs: + resources: + - name: reports + type: reports + params: + - filePattern: "test-report_*.json" + subFolder: "sonarExecuteScan" + - filePattern: "report1" + type: general + - name: commonPipelineEnvironment + type: piperEnvironment + params: + - name: artifactVersion + - name: git/commitId + - name: git/headCommitId + - name: git/branch + - name: custom/customList + type: "[]string" + - name: influxTest + type: influx + params: + - name: m1 + fields: + - name: f1 + tags: + - name: t1 + inputs: + resources: + - name: stashName + type: stash + params: + - name: param0 + aliases: + - name: oldparam0 + type: string + description: param0 description + default: val0 + scope: + - GENERAL + - PARAMETERS + mandatory: true + - name: param1 + aliases: + - name: oldparam1 + deprecated: true + type: string + description: param1 description + scope: + - PARAMETERS + possibleValues: + - value1 + - value2 + - value3 + deprecationMessage: use param3 instead + - name: param2 + type: string + description: param2 description + scope: + - PARAMETERS + mandatoryIf: + - name: param1 + value: value1 + - name: param3 + type: string + description: param3 description + scope: + - PARAMETERS + possibleValues: + - value1 + - value2 + - value3 + mandatoryIf: + - name: param1 + value: value1 + - name: param2 + value: value2 +` + var r string + switch name { + case "testStep.yaml": + r = meta1 + default: + r = "" + } + return io.NopCloser(strings.NewReader(r)), nil +} + +var files map[string][]byte + +func writeFileMock(filename string, data []byte, perm os.FileMode) error { + if files == nil { + files = make(map[string][]byte) + } + files[filename] = data + return nil +} + +func TestProcessMetaFiles(t *testing.T) { + + stepHelperData := StepHelperData{configOpenFileMock, writeFileMock, ""} + ProcessMetaFiles([]string{"testStep.yaml"}, "./cmd", stepHelperData) + + t.Run("step code", func(t *testing.T) { + goldenFilePath := filepath.Join("testdata", t.Name()+"_generated.golden") + expected, err := os.ReadFile(goldenFilePath) + if err != nil { + t.Fatalf("failed reading %v", goldenFilePath) + } + resultFilePath := filepath.Join("cmd", "testStep_generated.go") + assert.Equal(t, string(expected), string(files[resultFilePath])) + //t.Log(string(files[resultFilePath])) + }) + + t.Run("test code", func(t *testing.T) { + goldenFilePath := filepath.Join("testdata", t.Name()+"_generated.golden") + expected, err := os.ReadFile(goldenFilePath) + if err != nil { + t.Fatalf("failed reading %v", goldenFilePath) + } + resultFilePath := filepath.Join("cmd", "testStep_generated_test.go") + assert.Equal(t, string(expected), string(files[resultFilePath])) + }) + + t.Run("custom step code", func(t *testing.T) { + stepHelperData = StepHelperData{configOpenFileMock, writeFileMock, "piperOsCmd"} + ProcessMetaFiles([]string{"testStep.yaml"}, "./cmd", stepHelperData) + + goldenFilePath := filepath.Join("testdata", t.Name()+"_generated.golden") + expected, err := os.ReadFile(goldenFilePath) + if err != nil { + t.Fatalf("failed reading %v", goldenFilePath) + } + resultFilePath := filepath.Join("cmd", "testStep_generated.go") + assert.Equal(t, string(expected), string(files[resultFilePath])) + //t.Log(string(files[resultFilePath])) + }) +} + +func TestSetDefaultParameters(t *testing.T) { + t.Run("success case", func(t *testing.T) { + sliceVals := []string{"val4_1", "val4_2"} + stringSliceDefault := make([]interface{}, len(sliceVals)) + for i, v := range sliceVals { + stringSliceDefault[i] = v + } + stepData := config.StepData{ + Spec: config.StepSpec{ + Inputs: config.StepInputs{ + Parameters: []config.StepParameters{ + {Name: "param0", Type: "string", Default: "val0"}, + {Name: "param1", Type: "string"}, + {Name: "param2", Type: "bool", Default: true}, + {Name: "param3", Type: "bool"}, + {Name: "param4", Type: "[]string", Default: stringSliceDefault}, + {Name: "param5", Type: "[]string"}, + {Name: "param6", Type: "int"}, + {Name: "param7", Type: "int", Default: 1}, + }, + }, + }, + } + + expected := []string{ + "`val0`", + "os.Getenv(\"PIPER_param1\")", + "true", + "false", + "[]string{`val4_1`, `val4_2`}", + "[]string{}", + "0", + "1", + } + + osImport, err := setDefaultParameters(&stepData) + + assert.NoError(t, err, "error occurred but none expected") + + assert.Equal(t, true, osImport, "import of os package required") + + for k, v := range expected { + assert.Equal(t, v, stepData.Spec.Inputs.Parameters[k].Default, fmt.Sprintf("default not correct for parameter %v", k)) + } + }) + + t.Run("error case", func(t *testing.T) { + stepData := []config.StepData{ + { + Spec: config.StepSpec{ + Inputs: config.StepInputs{ + Parameters: []config.StepParameters{ + {Name: "param0", Type: "n/a", Default: 10}, + {Name: "param1", Type: "n/a"}, + }, + }, + }, + }, + { + Spec: config.StepSpec{ + Inputs: config.StepInputs{ + Parameters: []config.StepParameters{ + {Name: "param1", Type: "n/a"}, + }, + }, + }, + }, + } + + for k, v := range stepData { + _, err := setDefaultParameters(&v) + assert.Error(t, err, fmt.Sprintf("error expected but none occurred for parameter %v", k)) + } + }) +} + +func TestGetStepInfo(t *testing.T) { + + stepData := config.StepData{ + Metadata: config.StepMetadata{ + Name: "testStep", + Description: "Test description", + LongDescription: "Long Test description", + }, + Spec: config.StepSpec{ + Inputs: config.StepInputs{ + Parameters: []config.StepParameters{ + {Name: "param0", Scope: []string{"GENERAL"}, Type: "string", Default: "test"}, + }, + }, + }, + } + + myStepInfo, err := getStepInfo(&stepData, true, "") + + assert.NoError(t, err) + + assert.Equal(t, "testStep", myStepInfo.StepName, "StepName incorrect") + assert.Equal(t, "TestStepCommand", myStepInfo.CobraCmdFuncName, "CobraCmdFuncName incorrect") + assert.Equal(t, "createTestStepCmd", myStepInfo.CreateCmdVar, "CreateCmdVar incorrect") + assert.Equal(t, "Test description", myStepInfo.Short, "Short incorrect") + assert.Equal(t, "Long Test description", myStepInfo.Long, "Long incorrect") + assert.Equal(t, stepData.Spec.Inputs.Parameters, myStepInfo.StepParameters, "Metadata incorrect") + assert.Equal(t, "addTestStepFlags", myStepInfo.FlagsFunc, "FlagsFunc incorrect") + assert.Equal(t, "addTestStepFlags", myStepInfo.FlagsFunc, "FlagsFunc incorrect") + +} + +func TestLongName(t *testing.T) { + tt := []struct { + input string + expected string + }{ + {input: "my long name with no ticks", expected: "my long name with no ticks"}, + {input: "my long name with `ticks`", expected: "my long name with ` + \"`\" + `ticks` + \"`\" + `"}, + } + + for k, v := range tt { + assert.Equal(t, v.expected, longName(v.input), fmt.Sprintf("wrong long name for run %v", k)) + } +} + +func TestGolangNameTitle(t *testing.T) { + tt := []struct { + input string + expected string + }{ + {input: "testApi", expected: "TestAPI"}, + {input: "apiTest", expected: "APITest"}, + {input: "testUrl", expected: "TestURL"}, + {input: "testId", expected: "TestID"}, + {input: "testJson", expected: "TestJSON"}, + {input: "jsonTest", expected: "JSONTest"}, + } + + for k, v := range tt { + assert.Equal(t, v.expected, GolangNameTitle(v.input), fmt.Sprintf("wrong golang name for run %v", k)) + } +} + +func TestFlagType(t *testing.T) { + tt := []struct { + input string + expected string + }{ + {input: "bool", expected: "BoolVar"}, + {input: "int", expected: "IntVar"}, + {input: "string", expected: "StringVar"}, + {input: "[]string", expected: "StringSliceVar"}, + } + + for k, v := range tt { + assert.Equal(t, v.expected, flagType(v.input), fmt.Sprintf("wrong flag type for run %v", k)) + } +} + +func TestGetStringSliceFromInterface(t *testing.T) { + tt := []struct { + input interface{} + expected []string + }{ + {input: []interface{}{"Test", 2}, expected: []string{"Test", "2"}}, + {input: "Test", expected: []string{"Test"}}, + } + + for _, v := range tt { + assert.Equal(t, v.expected, getStringSliceFromInterface(v.input), "interface conversion failed") + } +} diff --git a/generator/helper/piper-context-defaults.yaml b/generator/helper/piper-context-defaults.yaml new file mode 100644 index 000000000..e2ef0d674 --- /dev/null +++ b/generator/helper/piper-context-defaults.yaml @@ -0,0 +1,167 @@ +metadata: + name: context defaults + description: These default descriptions will be used for the documentation generation of the pipeline steps. + longDescription: |- + These default descriptions will be used for the documentation generation of the pipeline steps for the context defaults. +spec: + inputs: + params: + - name: containerCommand + description: 'Kubernetes only: Allows to specify start command for container created with dockerImage parameter to overwrite Piper default (/usr/bin/tail -f /dev/null).' + type: string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: containerName + description: Optional configuration in combination with containerMap to define the container where the commands should be executed in. + type: string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: containerShell + description: Allows to specify the shell to be executed for container with containerName. + type: string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: dockerEnvVars + description: 'Environment variables to set in the container, e.g. [http_proxy: "proxy:8080"].' + type: map[string]string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: dockerImage + description: Name of the docker image that should be used. If empty, Docker is not used and the command is executed directly on the Jenkins system. + type: string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: dockerName + description: 'Kubernetes only: Name of the container launching dockerImage. SideCar only: Name of the container in local network.' + type: string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: dockerOptions + description: Docker options to be set when starting the container. + type: '[]string' + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: dockerPullImage + description: Set this to 'false' to bypass a docker image pull. Useful during development process. Allows testing of images which are available in the local registry only. + type: bool + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: dockerVolumeBind + description: Volumes that should be mounted into the docker container. + type: map[string]string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: dockerWorkspace + description: 'Kubernetes only: Specifies a dedicated user home directory for the container which will be passed as value for environment variable `HOME`.' + type: string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: sidecarCommand + description: Allows to specify a start command for the sidecar container. This parameter is similar to `containerCommand`. + type: string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: sidecarEnvVars + description: A map of environment variables to set in the sidecar container, similar to `dockerEnvVars`. + type: map[string]string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: sidecarImage + description: The name of the docker image of the sidecar container. If empty, no sidecar container is started. Similar to `dockerImage`. + type: string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: sidecarName + description: Name of the sidecar container. Similar to `dockerName`. + type: string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: sidecarPullImage + description: Set this to 'false' to bypass a docker image pull. Useful during development process. Allows testing of images which are available in the local registry only. + type: bool + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: sidecarReadyCommand + description: Command executed inside the container which returns exit code 0 when the container is ready to be used. + type: string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: sidecarOptions + description: Options to be set when starting the sidecar container. Similar to `dockerOptions`. + type: '[]string' + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: sidecarVolumeBind + description: Volumes that should be mounted into the sidecar container. Similar to `dockerVolumeBind`. + type: map[string]string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: sidecarWorkspace + type: string + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + - name: stashContent + description: Specific stashes that should be considered for the step execution. + type: '[]string' + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS diff --git a/generator/helper/resources.go b/generator/helper/resources.go new file mode 100644 index 000000000..80eca1399 --- /dev/null +++ b/generator/helper/resources.go @@ -0,0 +1,275 @@ +package helper + +import ( + "bytes" + "fmt" + "text/template" + + "github.com/SAP/jenkins-library/pkg/piperutils" +) + +// PiperEnvironmentResource defines a piper environement resource which stores data across multiple pipeline steps +type PiperEnvironmentResource struct { + Name string + StepName string + Parameters []PiperEnvironmentParameter + Categories []string +} + +// PiperEnvironmentParameter defines a parameter within the Piper environment +type PiperEnvironmentParameter struct { + Category string + Name string + Type string +} + +const piperEnvStructTemplate = `type {{ .StepName }}{{ .Name | title}} struct { + {{- range $notused, $param := .Parameters }} + {{- if not $param.Category}} + {{ $param.Name | golangName }} {{ $param.Type | resourceFieldType }} + {{- end }} + {{- end }} + {{- range $notused, $category := .Categories }} + {{ $category }} struct { + {{- range $notused, $param := $.Parameters }} + {{- if eq $category $param.Category }} + {{ $param.Name | golangName }} {{ $param.Type | resourceFieldType }} + {{- end }} + {{- end }} + } + {{- end }} +} + +func (p *{{ .StepName }}{{ .Name | title}}) persist(path, resourceName string) { + content := []struct{ + category string + name string + value interface{} + }{ + {{- range $notused, $param := .Parameters }} + {{- if not $param.Category}} + {category: "", name: "{{ $param.Name }}", value: p.{{ $param.Name | golangName}}}, + {{- else }} + {category: "{{ $param.Category }}", name: "{{ $param.Name }}", value: p.{{ $param.Category }}.{{ $param.Name | golangName}}}, + {{- end }} + {{- end }} + } + + errCount := 0 + for _, param := range content { + err := piperenv.SetResourceParameter(path, resourceName, filepath.Join(param.category, param.name), param.value) + if err != nil { + log.Entry().WithError(err).Error("Error persisting piper environment.") + errCount++ + } + } + if errCount > 0 { + log.Entry().Error("failed to persist Piper environment") + } +}` + +// StructName returns the name of the environment resource struct +func (p *PiperEnvironmentResource) StructName() string { + return fmt.Sprintf("%v%v", p.StepName, piperutils.Title(p.Name)) +} + +// StructString returns the golang coding for the struct definition of the environment resource +func (p *PiperEnvironmentResource) StructString() (string, error) { + funcMap := template.FuncMap{ + "title": piperutils.Title, + "golangName": golangName, + "resourceFieldType": resourceFieldType, + } + + tmpl, err := template.New("resources").Funcs(funcMap).Parse(piperEnvStructTemplate) + if err != nil { + return "", err + } + + var generatedCode bytes.Buffer + err = tmpl.Execute(&generatedCode, &p) + if err != nil { + return "", err + } + + return string(generatedCode.Bytes()), nil +} + +// InfluxResource defines an Influx resouece that holds measurement information for a pipeline run +type InfluxResource struct { + Name string + StepName string + Measurements []InfluxMeasurement +} + +// InfluxMeasurement defines a measurement for Influx reporting which is defined via a step resource +type InfluxMeasurement struct { + Name string + Fields []InfluxMetric + Tags []InfluxMetric +} + +// InfluxMetric defines a metric (column) in an influx measurement +type InfluxMetric struct { + Name string + Type string +} + +// InfluxMetricContent defines the content of an Inflx metric +type InfluxMetricContent struct { + Measurement string + ValType string + Name string + Value *string +} + +const influxStructTemplate = `type {{ .StepName }}{{ .Name | title}} struct { + {{- range $notused, $measurement := .Measurements }} + {{ $measurement.Name }} struct { + fields struct { + {{- range $notused, $field := $measurement.Fields }} + {{ $field.Name | golangName }} {{ $field.Type | resourceFieldType }} + {{- end }} + } + tags struct { + {{- range $notused, $tag := $measurement.Tags }} + {{ $tag.Name | golangName }} {{ $tag.Type | resourceFieldType }} + {{- end }} + } + } + {{- end }} +} + +func (i *{{ .StepName }}{{ .Name | title}}) persist(path, resourceName string) { + measurementContent := []struct{ + measurement string + valType string + name string + value interface{} + }{ + {{- range $notused, $measurement := .Measurements }} + {{- range $notused, $field := $measurement.Fields }} + {valType: config.InfluxField, measurement: "{{ $measurement.Name }}" , name: "{{ $field.Name }}", value: i.{{ $measurement.Name }}.fields.{{ $field.Name | golangName }}}, + {{- end }} + {{- range $notused, $tag := $measurement.Tags }} + {valType: config.InfluxTag, measurement: "{{ $measurement.Name }}" , name: "{{ $tag.Name }}", value: i.{{ $measurement.Name }}.tags.{{ $tag.Name | golangName }}}, + {{- end }} + {{- end }} + } + + errCount := 0 + for _, metric := range measurementContent { + err := piperenv.SetResourceParameter(path, resourceName, filepath.Join(metric.measurement, fmt.Sprintf("%vs", metric.valType), metric.name), metric.value) + if err != nil { + log.Entry().WithError(err).Error("Error persisting influx environment.") + errCount++ + } + } + if errCount > 0 { + log.Entry().Error("failed to persist Influx environment") + } +}` + +// StructString returns the golang coding for the struct definition of the InfluxResource +func (i *InfluxResource) StructString() (string, error) { + funcMap := template.FuncMap{ + "title": piperutils.Title, + "golangName": golangName, + "resourceFieldType": resourceFieldType, + } + + tmpl, err := template.New("resources").Funcs(funcMap).Parse(influxStructTemplate) + if err != nil { + return "", err + } + + var generatedCode bytes.Buffer + err = tmpl.Execute(&generatedCode, &i) + if err != nil { + return "", err + } + + return string(generatedCode.Bytes()), nil +} + +// StructName returns the name of the influx resource struct +func (i *InfluxResource) StructName() string { + return fmt.Sprintf("%v%v", i.StepName, piperutils.Title(i.Name)) +} + +// PiperEnvironmentResource defines a piper environement resource which stores data across multiple pipeline steps +type ReportsResource struct { + Name string + StepName string + Parameters []ReportsParameter +} + +// PiperEnvironmentParameter defines a parameter within the Piper environment +type ReportsParameter struct { + FilePattern string + ParamRef string + Type string +} + +const reportsStructTemplate = `type {{ .StepName }}{{ .Name | title}} struct { +} + +func (p *{{ .StepName }}{{ .Name | title}}) persist(stepConfig {{ .StepName }}Options, gcpJsonKeyFilePath string, gcsBucketId string, gcsFolderPath string, gcsSubFolder string) { + if gcsBucketId == "" { + log.Entry().Info("persisting reports to GCS is disabled, because gcsBucketId is empty") + return + } + log.Entry().Info("Uploading reports to Google Cloud Storage...") + content := []gcs.ReportOutputParam{ + {{- range $notused, $param := .Parameters }} + {FilePattern: "{{ $param.FilePattern }}", ParamRef: "{{ $param.ParamRef }}", StepResultType: "{{ $param.Type }}"}, + {{- end }} + } + + gcsClient, err := gcs.NewClient(gcpJsonKeyFilePath, "") + if err != nil { + log.Entry().Errorf("creation of GCS client failed: %v", err) + return + } + defer gcsClient.Close() + structVal := reflect.ValueOf(&stepConfig).Elem() + inputParameters := map[string]string{} + for i := 0; i < structVal.NumField(); i++ { + field := structVal.Type().Field(i) + if field.Type.String() == "string" { + paramName := strings.Split(field.Tag.Get("json"), ",") + paramValue, _ := structVal.Field(i).Interface().(string) + inputParameters[paramName[0]] = paramValue + } + } + if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, gcsFolderPath, gcsBucketId, gcsSubFolder, doublestar.Glob, os.Stat); err != nil { + log.Entry().Errorf("failed to persist reports: %v", err) + } +}` + +// StructName returns the name of the environment resource struct +func (p *ReportsResource) StructName() string { + return fmt.Sprintf("%v%v", p.StepName, piperutils.Title(p.Name)) +} + +// StructString returns the golang coding for the struct definition of the environment resource +func (p *ReportsResource) StructString() (string, error) { + funcMap := template.FuncMap{ + "title": piperutils.Title, + "golangName": golangName, + "resourceFieldType": resourceFieldType, + } + + tmpl, err := template.New("resources").Funcs(funcMap).Parse(reportsStructTemplate) + if err != nil { + return "", err + } + + var generatedCode bytes.Buffer + err = tmpl.Execute(&generatedCode, &p) + if err != nil { + return "", err + } + + return string(generatedCode.String()), nil +} diff --git a/generator/helper/resources_test.go b/generator/helper/resources_test.go new file mode 100644 index 000000000..846189745 --- /dev/null +++ b/generator/helper/resources_test.go @@ -0,0 +1,168 @@ +//go:build unit +// +build unit + +package helper + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInfluxResource_StructString(t *testing.T) { + tt := []struct { + in InfluxResource + expected string + }{ + { + in: InfluxResource{ + Name: "TestInflux", + StepName: "TestStep", + Measurements: []InfluxMeasurement{ + { + Name: "m1", + Fields: []InfluxMetric{{Name: "field1_1"}, {Name: "field1_2"}}, + Tags: []InfluxMetric{{Name: "tag1_1"}, {Name: "tag1_2"}}, + }, + { + Name: "m2", + Fields: []InfluxMetric{{Name: "field2_1"}, {Name: "field2_2"}}, + Tags: []InfluxMetric{{Name: "tag2_1"}, {Name: "tag2_2"}}, + }, + }, + }, + expected: `type TestStepTestInflux struct { + m1 struct { + fields struct { + field1_1 string + field1_2 string + } + tags struct { + tag1_1 string + tag1_2 string + } + } + m2 struct { + fields struct { + field2_1 string + field2_2 string + } + tags struct { + tag2_1 string + tag2_2 string + } + } +} + +func (i *TestStepTestInflux) persist(path, resourceName string) { + measurementContent := []struct{ + measurement string + valType string + name string + value interface{} + }{ + {valType: config.InfluxField, measurement: "m1" , name: "field1_1", value: i.m1.fields.field1_1}, + {valType: config.InfluxField, measurement: "m1" , name: "field1_2", value: i.m1.fields.field1_2}, + {valType: config.InfluxTag, measurement: "m1" , name: "tag1_1", value: i.m1.tags.tag1_1}, + {valType: config.InfluxTag, measurement: "m1" , name: "tag1_2", value: i.m1.tags.tag1_2}, + {valType: config.InfluxField, measurement: "m2" , name: "field2_1", value: i.m2.fields.field2_1}, + {valType: config.InfluxField, measurement: "m2" , name: "field2_2", value: i.m2.fields.field2_2}, + {valType: config.InfluxTag, measurement: "m2" , name: "tag2_1", value: i.m2.tags.tag2_1}, + {valType: config.InfluxTag, measurement: "m2" , name: "tag2_2", value: i.m2.tags.tag2_2}, + } + + errCount := 0 + for _, metric := range measurementContent { + err := piperenv.SetResourceParameter(path, resourceName, filepath.Join(metric.measurement, fmt.Sprintf("%vs", metric.valType), metric.name), metric.value) + if err != nil { + log.Entry().WithError(err).Error("Error persisting influx environment.") + errCount++ + } + } + if errCount > 0 { + log.Entry().Error("failed to persist Influx environment") + } +}`, + }, + } + + for run, test := range tt { + t.Run(fmt.Sprintf("Run %v", run), func(t *testing.T) { + got, err := test.in.StructString() + assert.NoError(t, err) + assert.Equal(t, test.expected, got) + }) + + } +} + +func TestReportsResource_StructString(t *testing.T) { + tt := []struct { + in ReportsResource + expected string + }{ + { + in: ReportsResource{ + Name: "reports", + StepName: "testStep", + Parameters: []ReportsParameter{ + { + FilePattern: "pattern1", + Type: "general", + }, + { + FilePattern: "pattern2", + }, + { + ParamRef: "testParam", + }, + }, + }, + expected: `type testStepReports struct { +} + +func (p *testStepReports) persist(stepConfig testStepOptions, gcpJsonKeyFilePath string, gcsBucketId string, gcsFolderPath string, gcsSubFolder string) { + if gcsBucketId == "" { + log.Entry().Info("persisting reports to GCS is disabled, because gcsBucketId is empty") + return + } + log.Entry().Info("Uploading reports to Google Cloud Storage...") + content := []gcs.ReportOutputParam{ + {FilePattern: "pattern1", ParamRef: "", StepResultType: "general"}, + {FilePattern: "pattern2", ParamRef: "", StepResultType: ""}, + {FilePattern: "", ParamRef: "testParam", StepResultType: ""}, + } + + gcsClient, err := gcs.NewClient(gcpJsonKeyFilePath, "") + if err != nil { + log.Entry().Errorf("creation of GCS client failed: %v", err) + return + } + defer gcsClient.Close() + structVal := reflect.ValueOf(&stepConfig).Elem() + inputParameters := map[string]string{} + for i := 0; i < structVal.NumField(); i++ { + field := structVal.Type().Field(i) + if field.Type.String() == "string" { + paramName := strings.Split(field.Tag.Get("json"), ",") + paramValue, _ := structVal.Field(i).Interface().(string) + inputParameters[paramName[0]] = paramValue + } + } + if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, gcsFolderPath, gcsBucketId, gcsSubFolder, doublestar.Glob, os.Stat); err != nil { + log.Entry().Errorf("failed to persist reports: %v", err) + } +}`, + }, + } + + for run, test := range tt { + t.Run(fmt.Sprintf("Run %v", run), func(t *testing.T) { + got, err := test.in.StructString() + assert.NoError(t, err) + assert.Equal(t, test.expected, got) + }) + + } +} diff --git a/generator/helper/testdata/TestProcessMetaFiles/README.md b/generator/helper/testdata/TestProcessMetaFiles/README.md new file mode 100644 index 000000000..9c0c73780 --- /dev/null +++ b/generator/helper/testdata/TestProcessMetaFiles/README.md @@ -0,0 +1 @@ +The `*_code_generated.golden` files need to be adapted for the changes as they are not generated tests. diff --git a/generator/helper/testdata/TestProcessMetaFiles/custom_step_code_generated.golden b/generator/helper/testdata/TestProcessMetaFiles/custom_step_code_generated.golden new file mode 100644 index 000000000..359eaad84 --- /dev/null +++ b/generator/helper/testdata/TestProcessMetaFiles/custom_step_code_generated.golden @@ -0,0 +1,365 @@ +// Code generated by piper's step-generator. DO NOT EDIT. + +package cmd + +import ( + "fmt" + "os" + "reflect" + "strings" + "path/filepath" + "time" + + piperOsCmd "github.com/SAP/jenkins-library/cmd" + "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/bmatcuk/doublestar" + "github.com/SAP/jenkins-library/pkg/gcs" + "github.com/SAP/jenkins-library/pkg/piperenv" + "github.com/SAP/jenkins-library/pkg/gcp" + "github.com/SAP/jenkins-library/pkg/telemetry" + "github.com/SAP/jenkins-library/pkg/splunk" + "github.com/SAP/jenkins-library/pkg/validation" + "github.com/spf13/cobra" +) + +type testStepOptions struct { + Param0 string `json:"param0,omitempty"` + Param1 string `json:"param1,omitempty" validate:"possible-values=value1 value2 value3"` + Param2 string `json:"param2,omitempty" validate:"required_if=Param1 value1"` + Param3 string `json:"param3,omitempty" validate:"possible-values=value1 value2 value3,required_if=Param1 value1 Param2 value2"` +} + + +type testStepReports struct { +} + +func (p *testStepReports) persist(stepConfig testStepOptions, gcpJsonKeyFilePath string, gcsBucketId string, gcsFolderPath string, gcsSubFolder string) { + if gcsBucketId == "" { + log.Entry().Info("persisting reports to GCS is disabled, because gcsBucketId is empty") + return + } + log.Entry().Info("Uploading reports to Google Cloud Storage...") + content := []gcs.ReportOutputParam{ + {FilePattern: "test-report_*.json", ParamRef: "", StepResultType: ""}, + {FilePattern: "report1", ParamRef: "", StepResultType: "general"}, + } + + gcsClient, err := gcs.NewClient(gcpJsonKeyFilePath, "") + if err != nil { + log.Entry().Errorf("creation of GCS client failed: %v", err) + return + } + defer gcsClient.Close() + structVal := reflect.ValueOf(&stepConfig).Elem() + inputParameters := map[string]string{} + for i := 0; i < structVal.NumField(); i++ { + field := structVal.Type().Field(i) + if field.Type.String() == "string" { + paramName := strings.Split(field.Tag.Get("json"), ",") + paramValue, _ := structVal.Field(i).Interface().(string) + inputParameters[paramName[0]] = paramValue + } + } + if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, gcsFolderPath, gcsBucketId, gcsSubFolder, doublestar.Glob, os.Stat); err != nil { + log.Entry().Errorf("failed to persist reports: %v", err) + } +} + +type testStepCommonPipelineEnvironment struct { + artifactVersion string + git struct { + commitID string + headCommitID string + branch string + } + custom struct { + customList []string + } +} + +func (p *testStepCommonPipelineEnvironment) persist(path, resourceName string) { + content := []struct{ + category string + name string + value interface{} + }{ + {category: "", name: "artifactVersion", value: p.artifactVersion}, + {category: "git", name: "commitId", value: p.git.commitID}, + {category: "git", name: "headCommitId", value: p.git.headCommitID}, + {category: "git", name: "branch", value: p.git.branch}, + {category: "custom", name: "customList", value: p.custom.customList}, + } + + errCount := 0 + for _, param := range content { + err := piperenv.SetResourceParameter(path, resourceName, filepath.Join(param.category, param.name), param.value) + if err != nil { + log.Entry().WithError(err).Error("Error persisting piper environment.") + errCount++ + } + } + if errCount > 0 { + log.Entry().Error("failed to persist Piper environment") + } +} + +type testStepInfluxTest struct { + m1 struct { + fields struct { + f1 string + } + tags struct { + t1 string + } + } +} + +func (i *testStepInfluxTest) persist(path, resourceName string) { + measurementContent := []struct{ + measurement string + valType string + name string + value interface{} + }{ + {valType: config.InfluxField, measurement: "m1" , name: "f1", value: i.m1.fields.f1}, + {valType: config.InfluxTag, measurement: "m1" , name: "t1", value: i.m1.tags.t1}, + } + + errCount := 0 + for _, metric := range measurementContent { + err := piperenv.SetResourceParameter(path, resourceName, filepath.Join(metric.measurement, fmt.Sprintf("%vs", metric.valType), metric.name), metric.value) + if err != nil { + log.Entry().WithError(err).Error("Error persisting influx environment.") + errCount++ + } + } + if errCount > 0 { + log.Entry().Error("failed to persist Influx environment") + } +} + + +// TestStepCommand Test description +func TestStepCommand() *cobra.Command { + const STEP_NAME = "testStep" + + metadata := testStepMetadata() + var stepConfig testStepOptions + var startTime time.Time + var reports testStepReports + var commonPipelineEnvironment testStepCommonPipelineEnvironment + var influxTest testStepInfluxTest + var logCollector *log.CollectorHook + var splunkClient *splunk.Splunk + telemetryClient := &telemetry.Telemetry{} + + var createTestStepCmd = &cobra.Command{ + Use: STEP_NAME, + Short: "Test description", + Long: `Long Test description`, + PreRunE: func(cmd *cobra.Command, _ []string) error { + startTime = time.Now() + log.SetStepName(STEP_NAME) + log.SetVerbose(piperOsCmd.GeneralConfig.Verbose) + + piperOsCmd.GeneralConfig.GitHubAccessTokens = piperOsCmd.ResolveAccessTokens(piperOsCmd.GeneralConfig.GitHubTokens) + + path, err := os.Getwd() + if err != nil { + return err + } + fatalHook := &log.FatalHook{CorrelationID: piperOsCmd.GeneralConfig.CorrelationID, Path: path} + log.RegisterHook(fatalHook) + + err = piperOsCmd.PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + if err != nil { + log.SetErrorCategory(log.ErrorConfiguration) + return err + } + + if len(piperOsCmd.GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 { + sentryHook := log.NewSentryHook(piperOsCmd.GeneralConfig.HookConfig.SentryConfig.Dsn, piperOsCmd.GeneralConfig.CorrelationID) + log.RegisterHook(&sentryHook) + } + + if len(piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 || len(piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.ProdCriblEndpoint) > 0 { + splunkClient = &splunk.Splunk{} + logCollector = &log.CollectorHook{CorrelationID: piperOsCmd.GeneralConfig.CorrelationID} + log.RegisterHook(logCollector) + } + + if err = log.RegisterANSHookIfConfigured(piperOsCmd.GeneralConfig.CorrelationID); err != nil { + log.Entry().WithError(err).Warn("failed to set up SAP Alert Notification Service log hook") + } + + validation, err := validation.New(validation.WithJSONNamesForStructFields(), validation.WithPredefinedErrorMessages()) + if err != nil { + return err + } + if err = validation.ValidateStruct(stepConfig); err != nil { + log.SetErrorCategory(log.ErrorConfiguration) + return err + } + + return nil + }, + Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + + stepTelemetryData := telemetry.CustomData{} + stepTelemetryData.ErrorCode = "1" + handler := func() { + reports.persist(stepConfig,piperOsCmd.GeneralConfig.GCPJsonKeyFilePath,piperOsCmd.GeneralConfig.GCSBucketId,piperOsCmd.GeneralConfig.GCSFolderPath,piperOsCmd.GeneralConfig.GCSSubFolder) + commonPipelineEnvironment.persist(piperOsCmd.GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + influxTest.persist(piperOsCmd.GeneralConfig.EnvRootPath, "influxTest") + config.RemoveVaultSecretFiles() + stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) + stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() + stepTelemetryData.PiperCommitHash = piperOsCmd.GitCommit + telemetryClient.SetData(&stepTelemetryData) + telemetryClient.LogStepTelemetryData() + if len(piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 { + splunkClient.Initialize(piperOsCmd.GeneralConfig.CorrelationID, + piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.Dsn, + piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.Token, + piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.Index, + piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.SendLogs) + splunkClient.Send(telemetryClient.GetData(), logCollector) + } + if len(piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.ProdCriblEndpoint) > 0 { + splunkClient.Initialize(piperOsCmd.GeneralConfig.CorrelationID, + piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.ProdCriblEndpoint, + piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.ProdCriblToken, + piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.ProdCriblIndex, + piperOsCmd.GeneralConfig.HookConfig.SplunkConfig.SendLogs) + splunkClient.Send(telemetryClient.GetData(), logCollector) + } + if piperOsCmd.GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + piperOsCmd.GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + piperOsCmd.GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + piperOsCmd.GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + piperOsCmd.GeneralConfig.CorrelationID, + piperOsCmd.GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(piperOsCmd.GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } + } + log.DeferExitHandler(handler) + defer handler() + telemetryClient.Initialize(STEP_NAME) + testStep(stepConfig, &stepTelemetryData, &commonPipelineEnvironment, &influxTest) + stepTelemetryData.ErrorCode = "0" + log.Entry().Info("SUCCESS") + }, + } + + addTestStepFlags(createTestStepCmd, &stepConfig) + return createTestStepCmd +} + +func addTestStepFlags(cmd *cobra.Command, stepConfig *testStepOptions) { + cmd.Flags().StringVar(&stepConfig.Param0, "param0", `val0`, "param0 description") + cmd.Flags().StringVar(&stepConfig.Param1, "param1", os.Getenv("PIPER_param1"), "param1 description") + cmd.Flags().StringVar(&stepConfig.Param2, "param2", os.Getenv("PIPER_param2"), "param2 description") + cmd.Flags().StringVar(&stepConfig.Param3, "param3", os.Getenv("PIPER_param3"), "param3 description") + + cmd.MarkFlagRequired("param0") + cmd.Flags().MarkDeprecated("param1", "use param3 instead") +} + +// retrieve step metadata +func testStepMetadata() config.StepData { + var theMetaData = config.StepData{ + Metadata: config.StepMetadata{ + Name: "testStep", + Aliases: []config.Alias{{Name: "testStepAlias", Deprecated: true},}, + Description: "Test description", + }, + Spec: config.StepSpec{ + Inputs: config.StepInputs{ + Resources: []config.StepResources{ + {Name: "stashName",Type: "stash", + }, + }, + Parameters: []config.StepParameters{ + { + Name: "param0", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"GENERAL","PARAMETERS",}, + Type: "string", + Mandatory: true, + Aliases: []config.Alias{{Name: "oldparam0"},}, + Default: `val0`, + }, + { + Name: "param1", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS",}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{{Name: "oldparam1", Deprecated: true},}, + Default: os.Getenv("PIPER_param1"), + DeprecationMessage: "use param3 instead", + }, + { + Name: "param2", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS",}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_param2"), + }, + { + Name: "param3", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS",}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_param3"), + }, + }, + }, + Outputs: config.StepOutputs{ + Resources: []config.StepResources{ + { + Name: "reports", + Type: "reports", + Parameters: []map[string]interface{}{ + {"filePattern": "test-report_*.json","subFolder": "sonarExecuteScan",}, + {"filePattern": "report1","type": "general",}, + }, + }, + { + Name: "commonPipelineEnvironment", + Type: "piperEnvironment", + Parameters: []map[string]interface{}{ + {"name": "artifactVersion",}, + {"name": "git/commitId",}, + {"name": "git/headCommitId",}, + {"name": "git/branch",}, + {"name": "custom/customList","type": "[]string",}, + }, + }, + { + Name: "influxTest", + Type: "influx", + Parameters: []map[string]interface{}{ + {"name": "m1","fields": []map[string]string{ {"name": "f1"}, },"tags": []map[string]string{ {"name": "t1"}, },}, + }, + }, + }, + }, + }, + } + return theMetaData +} diff --git a/generator/helper/testdata/TestProcessMetaFiles/step_code_generated.golden b/generator/helper/testdata/TestProcessMetaFiles/step_code_generated.golden new file mode 100644 index 000000000..28615caf7 --- /dev/null +++ b/generator/helper/testdata/TestProcessMetaFiles/step_code_generated.golden @@ -0,0 +1,364 @@ +// Code generated by piper's step-generator. DO NOT EDIT. + +package cmd + +import ( + "fmt" + "os" + "reflect" + "strings" + "path/filepath" + "time" + + "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/bmatcuk/doublestar" + "github.com/SAP/jenkins-library/pkg/gcs" + "github.com/SAP/jenkins-library/pkg/piperenv" + "github.com/SAP/jenkins-library/pkg/gcp" + "github.com/SAP/jenkins-library/pkg/telemetry" + "github.com/SAP/jenkins-library/pkg/splunk" + "github.com/SAP/jenkins-library/pkg/validation" + "github.com/spf13/cobra" +) + +type testStepOptions struct { + Param0 string `json:"param0,omitempty"` + Param1 string `json:"param1,omitempty" validate:"possible-values=value1 value2 value3"` + Param2 string `json:"param2,omitempty" validate:"required_if=Param1 value1"` + Param3 string `json:"param3,omitempty" validate:"possible-values=value1 value2 value3,required_if=Param1 value1 Param2 value2"` +} + + +type testStepReports struct { +} + +func (p *testStepReports) persist(stepConfig testStepOptions, gcpJsonKeyFilePath string, gcsBucketId string, gcsFolderPath string, gcsSubFolder string) { + if gcsBucketId == "" { + log.Entry().Info("persisting reports to GCS is disabled, because gcsBucketId is empty") + return + } + log.Entry().Info("Uploading reports to Google Cloud Storage...") + content := []gcs.ReportOutputParam{ + {FilePattern: "test-report_*.json", ParamRef: "", StepResultType: ""}, + {FilePattern: "report1", ParamRef: "", StepResultType: "general"}, + } + + gcsClient, err := gcs.NewClient(gcpJsonKeyFilePath, "") + if err != nil { + log.Entry().Errorf("creation of GCS client failed: %v", err) + return + } + defer gcsClient.Close() + structVal := reflect.ValueOf(&stepConfig).Elem() + inputParameters := map[string]string{} + for i := 0; i < structVal.NumField(); i++ { + field := structVal.Type().Field(i) + if field.Type.String() == "string" { + paramName := strings.Split(field.Tag.Get("json"), ",") + paramValue, _ := structVal.Field(i).Interface().(string) + inputParameters[paramName[0]] = paramValue + } + } + if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, gcsFolderPath, gcsBucketId, gcsSubFolder, doublestar.Glob, os.Stat); err != nil { + log.Entry().Errorf("failed to persist reports: %v", err) + } +} + +type testStepCommonPipelineEnvironment struct { + artifactVersion string + git struct { + commitID string + headCommitID string + branch string + } + custom struct { + customList []string + } +} + +func (p *testStepCommonPipelineEnvironment) persist(path, resourceName string) { + content := []struct{ + category string + name string + value interface{} + }{ + {category: "", name: "artifactVersion", value: p.artifactVersion}, + {category: "git", name: "commitId", value: p.git.commitID}, + {category: "git", name: "headCommitId", value: p.git.headCommitID}, + {category: "git", name: "branch", value: p.git.branch}, + {category: "custom", name: "customList", value: p.custom.customList}, + } + + errCount := 0 + for _, param := range content { + err := piperenv.SetResourceParameter(path, resourceName, filepath.Join(param.category, param.name), param.value) + if err != nil { + log.Entry().WithError(err).Error("Error persisting piper environment.") + errCount++ + } + } + if errCount > 0 { + log.Entry().Error("failed to persist Piper environment") + } +} + +type testStepInfluxTest struct { + m1 struct { + fields struct { + f1 string + } + tags struct { + t1 string + } + } +} + +func (i *testStepInfluxTest) persist(path, resourceName string) { + measurementContent := []struct{ + measurement string + valType string + name string + value interface{} + }{ + {valType: config.InfluxField, measurement: "m1" , name: "f1", value: i.m1.fields.f1}, + {valType: config.InfluxTag, measurement: "m1" , name: "t1", value: i.m1.tags.t1}, + } + + errCount := 0 + for _, metric := range measurementContent { + err := piperenv.SetResourceParameter(path, resourceName, filepath.Join(metric.measurement, fmt.Sprintf("%vs", metric.valType), metric.name), metric.value) + if err != nil { + log.Entry().WithError(err).Error("Error persisting influx environment.") + errCount++ + } + } + if errCount > 0 { + log.Entry().Error("failed to persist Influx environment") + } +} + + +// TestStepCommand Test description +func TestStepCommand() *cobra.Command { + const STEP_NAME = "testStep" + + metadata := testStepMetadata() + var stepConfig testStepOptions + var startTime time.Time + var reports testStepReports + var commonPipelineEnvironment testStepCommonPipelineEnvironment + var influxTest testStepInfluxTest + var logCollector *log.CollectorHook + var splunkClient *splunk.Splunk + telemetryClient := &telemetry.Telemetry{} + + var createTestStepCmd = &cobra.Command{ + Use: STEP_NAME, + Short: "Test description", + Long: `Long Test description`, + PreRunE: func(cmd *cobra.Command, _ []string) error { + startTime = time.Now() + log.SetStepName(STEP_NAME) + log.SetVerbose(GeneralConfig.Verbose) + + GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) + + path, err := os.Getwd() + if err != nil { + return err + } + fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} + log.RegisterHook(fatalHook) + + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + if err != nil { + log.SetErrorCategory(log.ErrorConfiguration) + return err + } + + if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 { + sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID) + log.RegisterHook(&sentryHook) + } + + if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 || len(GeneralConfig.HookConfig.SplunkConfig.ProdCriblEndpoint) > 0 { + splunkClient = &splunk.Splunk{} + logCollector = &log.CollectorHook{CorrelationID: GeneralConfig.CorrelationID} + log.RegisterHook(logCollector) + } + + if err = log.RegisterANSHookIfConfigured(GeneralConfig.CorrelationID); err != nil { + log.Entry().WithError(err).Warn("failed to set up SAP Alert Notification Service log hook") + } + + validation, err := validation.New(validation.WithJSONNamesForStructFields(), validation.WithPredefinedErrorMessages()) + if err != nil { + return err + } + if err = validation.ValidateStruct(stepConfig); err != nil { + log.SetErrorCategory(log.ErrorConfiguration) + return err + } + + return nil + }, + Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + + stepTelemetryData := telemetry.CustomData{} + stepTelemetryData.ErrorCode = "1" + handler := func() { + reports.persist(stepConfig,GeneralConfig.GCPJsonKeyFilePath,GeneralConfig.GCSBucketId,GeneralConfig.GCSFolderPath,GeneralConfig.GCSSubFolder) + commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + influxTest.persist(GeneralConfig.EnvRootPath, "influxTest") + config.RemoveVaultSecretFiles() + stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) + stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() + stepTelemetryData.PiperCommitHash = GitCommit + telemetryClient.SetData(&stepTelemetryData) + telemetryClient.LogStepTelemetryData() + if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 { + splunkClient.Initialize(GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.SplunkConfig.Dsn, + GeneralConfig.HookConfig.SplunkConfig.Token, + GeneralConfig.HookConfig.SplunkConfig.Index, + GeneralConfig.HookConfig.SplunkConfig.SendLogs) + splunkClient.Send(telemetryClient.GetData(), logCollector) + } + if len(GeneralConfig.HookConfig.SplunkConfig.ProdCriblEndpoint) > 0 { + splunkClient.Initialize(GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.SplunkConfig.ProdCriblEndpoint, + GeneralConfig.HookConfig.SplunkConfig.ProdCriblToken, + GeneralConfig.HookConfig.SplunkConfig.ProdCriblIndex, + GeneralConfig.HookConfig.SplunkConfig.SendLogs) + splunkClient.Send(telemetryClient.GetData(), logCollector) + } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } + } + log.DeferExitHandler(handler) + defer handler() + telemetryClient.Initialize(STEP_NAME) + testStep(stepConfig, &stepTelemetryData, &commonPipelineEnvironment, &influxTest) + stepTelemetryData.ErrorCode = "0" + log.Entry().Info("SUCCESS") + }, + } + + addTestStepFlags(createTestStepCmd, &stepConfig) + return createTestStepCmd +} + +func addTestStepFlags(cmd *cobra.Command, stepConfig *testStepOptions) { + cmd.Flags().StringVar(&stepConfig.Param0, "param0", `val0`, "param0 description") + cmd.Flags().StringVar(&stepConfig.Param1, "param1", os.Getenv("PIPER_param1"), "param1 description") + cmd.Flags().StringVar(&stepConfig.Param2, "param2", os.Getenv("PIPER_param2"), "param2 description") + cmd.Flags().StringVar(&stepConfig.Param3, "param3", os.Getenv("PIPER_param3"), "param3 description") + + cmd.MarkFlagRequired("param0") + cmd.Flags().MarkDeprecated("param1", "use param3 instead") +} + +// retrieve step metadata +func testStepMetadata() config.StepData { + var theMetaData = config.StepData{ + Metadata: config.StepMetadata{ + Name: "testStep", + Aliases: []config.Alias{{Name: "testStepAlias", Deprecated: true},}, + Description: "Test description", + }, + Spec: config.StepSpec{ + Inputs: config.StepInputs{ + Resources: []config.StepResources{ + {Name: "stashName",Type: "stash", + }, + }, + Parameters: []config.StepParameters{ + { + Name: "param0", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"GENERAL","PARAMETERS",}, + Type: "string", + Mandatory: true, + Aliases: []config.Alias{{Name: "oldparam0"},}, + Default: `val0`, + }, + { + Name: "param1", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS",}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{{Name: "oldparam1", Deprecated: true},}, + Default: os.Getenv("PIPER_param1"), + DeprecationMessage: "use param3 instead", + }, + { + Name: "param2", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS",}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_param2"), + }, + { + Name: "param3", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS",}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_param3"), + }, + }, + }, + Outputs: config.StepOutputs{ + Resources: []config.StepResources{ + { + Name: "reports", + Type: "reports", + Parameters: []map[string]interface{}{ + {"filePattern": "test-report_*.json","subFolder": "sonarExecuteScan",}, + {"filePattern": "report1","type": "general",}, + }, + }, + { + Name: "commonPipelineEnvironment", + Type: "piperEnvironment", + Parameters: []map[string]interface{}{ + {"name": "artifactVersion",}, + {"name": "git/commitId",}, + {"name": "git/headCommitId",}, + {"name": "git/branch",}, + {"name": "custom/customList","type": "[]string",}, + }, + }, + { + Name: "influxTest", + Type: "influx", + Parameters: []map[string]interface{}{ + {"name": "m1","fields": []map[string]string{ {"name": "f1"}, },"tags": []map[string]string{ {"name": "t1"}, },}, + }, + }, + }, + }, + }, + } + return theMetaData +} diff --git a/generator/helper/testdata/TestProcessMetaFiles/test_code_generated.golden b/generator/helper/testdata/TestProcessMetaFiles/test_code_generated.golden new file mode 100644 index 000000000..5dcb76ff9 --- /dev/null +++ b/generator/helper/testdata/TestProcessMetaFiles/test_code_generated.golden @@ -0,0 +1,20 @@ +//go:build unit +// +build unit + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTestStepCommand(t *testing.T) { + t.Parallel() + + testCmd := TestStepCommand() + + // only high level testing performed - details are tested in step generation procedure + assert.Equal(t, "testStep", testCmd.Use, "command name incorrect") + +} diff --git a/generator/main.go b/generator/main.go new file mode 100644 index 000000000..54e7164f6 --- /dev/null +++ b/generator/main.go @@ -0,0 +1,67 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "os" + "os/exec" + + "github.com/SAP/jenkins-library/generator/helper" +) + +func main() { + metadataFile := *flag.String("metadataFile", "", "Single metadata file used to generate code for a step.") + metadataPath := *flag.String("metadataDir", "./resources/metadata", "The directory containing the step metadata. Default points to \\'resources/metadata\\'.") + targetDir := *flag.String("targetDir", "./cmd", "The target directory for the generated commands.") + flag.Parse() + fmt.Printf("metadataFile: %v\n,metadataDir: %v\n, targetDir: %v\n", metadataFile, metadataPath, targetDir) + + var metadataFiles []string + var err error + if metadataFile != "" { + fmt.Printf("Using single metadata file: %v\n", metadataFile) + metadataFiles = []string{metadataFile} + } else { + fmt.Printf("Using metadata directory: %v\n", metadataPath) + metadataFiles, err = helper.MetadataFiles(metadataPath) + if err != nil { + fmt.Printf("Error occurred: %v\n", err) + os.Exit(1) + } + } + + err = helper.ProcessMetaFiles(metadataFiles, targetDir, helper.StepHelperData{ + OpenFile: openMetaFile, + WriteFile: os.WriteFile, + ExportPrefix: "", + }) + if err != nil { + fmt.Printf("Error occurred: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Running go fmt %v\n", targetDir) + cmd := exec.Command("go", "fmt", targetDir) + r, _ := cmd.StdoutPipe() + cmd.Stderr = cmd.Stdout + done := make(chan struct{}) + scanner := bufio.NewScanner(r) + go func() { + for scanner.Scan() { + fmt.Println(scanner.Text()) + } + done <- struct{}{} + + }() + err = cmd.Run() + if err != nil { + fmt.Printf("Error occurred: %v\n", err) + os.Exit(1) + } +} + +func openMetaFile(name string) (io.ReadCloser, error) { + return os.Open(name) +}