package config import ( "bytes" "encoding/json" "fmt" "io" "io/ioutil" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/ghodss/yaml" "github.com/pkg/errors" ) // StepData defines the metadata for a step, like step descriptions, parameters, ... type StepData struct { Metadata StepMetadata `json:"metadata"` Spec StepSpec `json:"spec"` } // StepMetadata defines the metadata for a step, like step descriptions, parameters, ... type StepMetadata struct { Name string `json:"name"` Aliases []Alias `json:"aliases,omitempty"` Description string `json:"description"` LongDescription string `json:"longDescription,omitempty"` } // StepSpec defines the spec details for a step, like step inputs, containers, sidecars, ... type StepSpec struct { Inputs StepInputs `json:"inputs,omitempty"` Outputs StepOutputs `json:"outputs,omitempty"` Containers []Container `json:"containers,omitempty"` Sidecars []Container `json:"sidecars,omitempty"` } // StepInputs defines the spec details for a step, like step inputs, containers, sidecars, ... type StepInputs struct { Parameters []StepParameters `json:"params"` Resources []StepResources `json:"resources,omitempty"` Secrets []StepSecrets `json:"secrets,omitempty"` } // StepParameters defines the parameters for a step type StepParameters struct { Name string `json:"name"` Description string `json:"description"` LongDescription string `json:"longDescription,omitempty"` ResourceRef []ResourceReference `json:"resourceRef,omitempty"` Scope []string `json:"scope"` Type string `json:"type"` Mandatory bool `json:"mandatory,omitempty"` Default interface{} `json:"default,omitempty"` PossibleValues []interface{} `json:"possibleValues,omitempty"` Aliases []Alias `json:"aliases,omitempty"` Conditions []Condition `json:"conditions,omitempty"` Secret bool `json:"secret,omitempty"` MandatoryIf []ParameterDependence `json:"mandatoryIf,omitempty"` DeprecationMessage string `json:"deprecationMessage,omitempty"` } type ParameterDependence struct { Name string `json:"name"` Value string `json:"value"` } // ResourceReference defines the parameters of a resource reference type ResourceReference struct { Name string `json:"name"` Type string `json:"type,omitempty"` Param string `json:"param,omitempty"` Default string `json:"default,omitempty"` Aliases []Alias `json:"aliases,omitempty"` } // Alias defines a step input parameter alias type Alias struct { Name string `json:"name,omitempty"` Deprecated bool `json:"deprecated,omitempty"` } // StepResources defines the resources to be provided by the step context, e.g. Jenkins pipeline type StepResources struct { Name string `json:"name"` Description string `json:"description,omitempty"` Type string `json:"type,omitempty"` Parameters []map[string]interface{} `json:"params,omitempty"` Conditions []Condition `json:"conditions,omitempty"` } // StepSecrets defines the secrets to be provided by the step context, e.g. Jenkins pipeline type StepSecrets struct { Name string `json:"name"` Description string `json:"description,omitempty"` Type string `json:"type,omitempty"` Aliases []Alias `json:"aliases,omitempty"` } // StepOutputs defines the outputs of a step step, typically one or multiple resources type StepOutputs struct { Resources []StepResources `json:"resources,omitempty"` } // Container defines an execution container type Container struct { //ToDo: check dockerOptions, dockerVolumeBind, containerPortMappings, sidecarOptions, sidecarVolumeBind Command []string `json:"command"` EnvVars []EnvVar `json:"env"` Image string `json:"image"` ImagePullPolicy string `json:"imagePullPolicy"` Name string `json:"name"` ReadyCommand string `json:"readyCommand"` Shell string `json:"shell"` WorkingDir string `json:"workingDir"` Conditions []Condition `json:"conditions,omitempty"` Options []Option `json:"options,omitempty"` //VolumeMounts []VolumeMount `json:"volumeMounts,omitempty"` } // ToDo: Add the missing Volumes part to enable the volume mount completely // VolumeMount defines a mount path // type VolumeMount struct { // MountPath string `json:"mountPath"` // Name string `json:"name"` //} // Option defines an docker option type Option struct { Name string `json:"name"` Value string `json:"value"` } // EnvVar defines an environment variable type EnvVar struct { Name string `json:"name"` Value string `json:"value"` } // Condition defines an condition which decides when the parameter, resource or container is valid type Condition struct { ConditionRef string `json:"conditionRef"` Params []Param `json:"params"` } // Param defines the parameters serving as inputs to the condition type Param struct { Name string `json:"name"` Value string `json:"value"` } // StepFilters defines the filter parameters for the different sections type StepFilters struct { All []string General []string Stages []string Steps []string Parameters []string Env []string } // ReadPipelineStepData loads step definition in yaml format func (m *StepData) ReadPipelineStepData(metadata io.ReadCloser) error { defer metadata.Close() content, err := ioutil.ReadAll(metadata) if err != nil { return errors.Wrapf(err, "error reading %v", metadata) } err = yaml.Unmarshal(content, &m) if err != nil { return errors.Wrapf(err, "error unmarshalling: %v", err) } return nil } // GetParameterFilters retrieves all scope dependent parameter filters func (m *StepData) GetParameterFilters() StepFilters { filters := StepFilters{All: []string{"verbose"}, General: []string{"verbose"}, Steps: []string{"verbose"}, Stages: []string{"verbose"}, Parameters: []string{"verbose"}} for _, param := range m.Spec.Inputs.Parameters { parameterKeys := []string{param.Name} for _, condition := range param.Conditions { for _, dependentParam := range condition.Params { parameterKeys = append(parameterKeys, dependentParam.Value) } } filters.All = append(filters.All, parameterKeys...) for _, scope := range param.Scope { switch scope { case "GENERAL": filters.General = append(filters.General, parameterKeys...) case "STEPS": filters.Steps = append(filters.Steps, parameterKeys...) case "STAGES": filters.Stages = append(filters.Stages, parameterKeys...) case "PARAMETERS": filters.Parameters = append(filters.Parameters, parameterKeys...) case "ENV": filters.Env = append(filters.Env, parameterKeys...) } } } return filters } // GetContextParameterFilters retrieves all scope dependent parameter filters func (m *StepData) GetContextParameterFilters() StepFilters { var filters StepFilters contextFilters := []string{} for _, secret := range m.Spec.Inputs.Secrets { contextFilters = append(contextFilters, secret.Name) } if len(m.Spec.Inputs.Resources) > 0 { for _, res := range m.Spec.Inputs.Resources { if res.Type == "stash" { contextFilters = append(contextFilters, "stashContent") break } } } if len(m.Spec.Containers) > 0 { parameterKeys := []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerName", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "dockerRegistryUrl", "dockerRegistryCredentialsId"} for _, container := range m.Spec.Containers { for _, condition := range container.Conditions { for _, dependentParam := range condition.Params { parameterKeys = append(parameterKeys, dependentParam.Value) parameterKeys = append(parameterKeys, dependentParam.Name) } } } // ToDo: append dependentParam.Value & dependentParam.Name only according to correct parameter scope and not generally contextFilters = append(contextFilters, parameterKeys...) } if len(m.Spec.Sidecars) > 0 { //ToDo: support fallback for "dockerName" configuration property -> via aliasing? contextFilters = append(contextFilters, []string{"containerName", "containerPortMappings", "dockerName", "sidecarEnvVars", "sidecarImage", "sidecarName", "sidecarOptions", "sidecarPullImage", "sidecarReadyCommand", "sidecarVolumeBind", "sidecarWorkspace"}...) //ToDo: add condition param.Value and param.Name to filter as for Containers } contextFilters = addVaultContextParametersFilter(m, contextFilters) if len(contextFilters) > 0 { filters.All = append(filters.All, contextFilters...) filters.General = append(filters.General, contextFilters...) filters.Steps = append(filters.Steps, contextFilters...) filters.Stages = append(filters.Stages, contextFilters...) filters.Parameters = append(filters.Parameters, contextFilters...) filters.Env = append(filters.Env, contextFilters...) } return filters } func addVaultContextParametersFilter(m *StepData, contextFilters []string) []string { contextFilters = append(contextFilters, []string{"vaultAppRoleTokenCredentialsId", "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}...) return contextFilters } // GetContextDefaults retrieves context defaults like container image, name, env vars, resources, ... // It only supports scenarios with one container and optionally one sidecar func (m *StepData) GetContextDefaults(stepName string) (io.ReadCloser, error) { //ToDo error handling empty Containers/Sidecars //ToDo handle empty Command root := map[string]interface{}{} if len(m.Spec.Containers) > 0 { for _, container := range m.Spec.Containers { key := "" conditionParam := "" if len(container.Conditions) > 0 { key = container.Conditions[0].Params[0].Value conditionParam = container.Conditions[0].Params[0].Name } p := map[string]interface{}{} if key != "" { root[key] = p //add default for condition parameter if available for _, inputParam := range m.Spec.Inputs.Parameters { if inputParam.Name == conditionParam { root[conditionParam] = inputParam.Default } } } else { p = root } if len(container.Command) > 0 { p["containerCommand"] = container.Command[0] } putStringIfNotEmpty(p, "containerName", container.Name) putStringIfNotEmpty(p, "containerShell", container.Shell) container.commonConfiguration("docker", &p) // Ready command not relevant for main runtime container so far //putStringIfNotEmpty(p, ..., container.ReadyCommand) } } if len(m.Spec.Sidecars) > 0 { if len(m.Spec.Sidecars[0].Command) > 0 { root["sidecarCommand"] = m.Spec.Sidecars[0].Command[0] } m.Spec.Sidecars[0].commonConfiguration("sidecar", &root) putStringIfNotEmpty(root, "sidecarReadyCommand", m.Spec.Sidecars[0].ReadyCommand) // not filled for now since this is not relevant in Kubernetes case //putStringIfNotEmpty(root, "containerPortMappings", m.Spec.Sidecars[0].) } if len(m.Spec.Inputs.Resources) > 0 { keys := []string{} resources := map[string][]string{} for _, resource := range m.Spec.Inputs.Resources { if resource.Type == "stash" { key := "" if len(resource.Conditions) > 0 { key = resource.Conditions[0].Params[0].Value } if resources[key] == nil { keys = append(keys, key) resources[key] = []string{} } resources[key] = append(resources[key], resource.Name) } } for _, key := range keys { if key == "" { root["stashContent"] = resources[""] } else { if root[key] == nil { root[key] = map[string]interface{}{ "stashContent": resources[key], } } else { p := root[key].(map[string]interface{}) p["stashContent"] = resources[key] } } } } c := Config{ Steps: map[string]map[string]interface{}{ stepName: root, }, } JSON, err := yaml.Marshal(c) if err != nil { return nil, errors.Wrap(err, "failed to create context defaults") } r := ioutil.NopCloser(bytes.NewReader(JSON)) return r, nil } // GetResourceParameters retrieves parameters from a named pipeline resource with a defined path func (m *StepData) GetResourceParameters(path, name string) map[string]interface{} { resourceParams := map[string]interface{}{} for _, param := range m.Spec.Inputs.Parameters { for _, res := range param.ResourceRef { if res.Name == name { if val := getParameterValue(path, res, param); val != nil { resourceParams[param.Name] = val break } } } } return resourceParams } func (container *Container) commonConfiguration(keyPrefix string, config *map[string]interface{}) { putMapIfNotEmpty(*config, keyPrefix+"EnvVars", EnvVarsAsMap(container.EnvVars)) putStringIfNotEmpty(*config, keyPrefix+"Image", container.Image) putStringIfNotEmpty(*config, keyPrefix+"Name", container.Name) if container.ImagePullPolicy != "" { (*config)[keyPrefix+"PullImage"] = container.ImagePullPolicy != "Never" } putStringIfNotEmpty(*config, keyPrefix+"Workspace", container.WorkingDir) putSliceIfNotEmpty(*config, keyPrefix+"Options", OptionsAsStringSlice(container.Options)) //putSliceIfNotEmpty(*config, keyPrefix+"VolumeBind", volumeMountsAsStringSlice(container.VolumeMounts)) } func getParameterValue(path string, res ResourceReference, param StepParameters) interface{} { paramName := res.Param if param.Type != "string" { paramName += ".json" } if val := piperenv.GetResourceParameter(path, res.Name, paramName); len(val) > 0 { if param.Type != "string" { var unmarshalledValue interface{} err := json.Unmarshal([]byte(val), &unmarshalledValue) if err != nil { log.Entry().Debugf("Failed to unmarshal: %v", val) } return unmarshalledValue } return val } return nil } // GetReference returns the ResourceReference of the given type func (m *StepParameters) GetReference(refType string) *ResourceReference { for _, ref := range m.ResourceRef { if refType == ref.Type { return &ref } } return nil } func getFilterForResourceReferences(params []StepParameters) []string { var filter []string for _, param := range params { reference := param.GetReference("vaultSecret") if reference == nil { reference = param.GetReference("vaultSecretFile") } if reference == nil { return filter } if reference.Name != "" { filter = append(filter, reference.Name) } } return filter } // HasReference checks whether StepData contains a parameter that has Reference with the given type func (m *StepData) HasReference(refType string) bool { for _, param := range m.Spec.Inputs.Parameters { if param.GetReference(refType) != nil { return true } } return false } // EnvVarsAsMap converts container EnvVars into a map as required by dockerExecute func EnvVarsAsMap(envVars []EnvVar) map[string]string { e := map[string]string{} for _, v := range envVars { e[v.Name] = v.Value } return e } // OptionsAsStringSlice converts container options into a string slice as required by dockerExecute func OptionsAsStringSlice(options []Option) []string { e := []string{} for _, v := range options { if len(v.Value) != 0 { e = append(e, fmt.Sprintf("%v %v", v.Name, v.Value)) } else { e = append(e, fmt.Sprintf("%v=", v.Name)) } } return e } func putStringIfNotEmpty(config map[string]interface{}, key, value string) { if value != "" { config[key] = value } } func putMapIfNotEmpty(config map[string]interface{}, key string, value map[string]string) { if len(value) > 0 { config[key] = value } } func putSliceIfNotEmpty(config map[string]interface{}, key string, value []string) { if len(value) > 0 { config[key] = value } } func ResolveMetadata(gitHubTokens map[string]string, metaDataResolver func() map[string]StepData, stepMetadata string, stepName string) (StepData, error) { var metadata StepData if stepMetadata != "" { metadataFile, err := OpenPiperFile(stepMetadata, gitHubTokens) if err != nil { return metadata, errors.Wrap(err, "open failed") } err = metadata.ReadPipelineStepData(metadataFile) if err != nil { return metadata, errors.Wrap(err, "read failed") } } else { if stepName != "" { if metaDataResolver == nil { return metadata, errors.New("metaDataResolver is nil") } metadataMap := metaDataResolver() var ok bool metadata, ok = metadataMap[stepName] if !ok { return metadata, errors.Errorf("could not retrieve by stepName %v", stepName) } } else { return metadata, errors.Errorf("either one of stepMetadata or stepName parameter has to be passed") } } return metadata, nil } //ToDo: Enable this when the Volumes part is also implemented //func volumeMountsAsStringSlice(volumeMounts []VolumeMount) []string { // e := []string{} // for _, v := range volumeMounts { // e = append(e, fmt.Sprintf("%v:%v", v.Name, v.MountPath)) // } // return e //}