package cmd import ( "fmt" "io" "os" "path" "path/filepath" "strings" "github.com/SAP/jenkins-library/pkg/config" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/reporting" ws "github.com/SAP/jenkins-library/pkg/whitesource" "github.com/pkg/errors" "github.com/spf13/cobra" ) type ConfigCommandOptions struct { Output string // output format, so far only JSON, YAML OutputFile string // if set: path to file where the output should be written to ParametersJSON string // parameters to be considered in JSON format StageConfig bool StageConfigAcceptedParameters []string StepMetadata string // metadata to be considered, can be filePath or ENV containing JSON in format 'ENV:MY_ENV_VAR' StepName string ContextConfig bool OpenFile func(s string, t map[string]string) (io.ReadCloser, error) } var configOptions ConfigCommandOptions func SetConfigOptions(c ConfigCommandOptions) { configOptions.ContextConfig = c.ContextConfig configOptions.OpenFile = c.OpenFile configOptions.Output = c.Output configOptions.OutputFile = c.OutputFile configOptions.ParametersJSON = c.ParametersJSON configOptions.StageConfig = c.StageConfig configOptions.StageConfigAcceptedParameters = c.StageConfigAcceptedParameters configOptions.StepMetadata = c.StepMetadata configOptions.StepName = c.StepName } type getConfigUtils interface { FileExists(filename string) (bool, error) DirExists(path string) (bool, error) FileWrite(path string, content []byte, perm os.FileMode) error } type getConfigUtilsBundle struct { *piperutils.Files } func newGetConfigUtilsUtils() getConfigUtils { return &getConfigUtilsBundle{ Files: &piperutils.Files{}, } } // ConfigCommand is the entry command for loading the configuration of a pipeline step func ConfigCommand() *cobra.Command { SetConfigOptions(ConfigCommandOptions{ OpenFile: config.OpenPiperFile, }) var createConfigCmd = &cobra.Command{ Use: "getConfig", Short: "Loads the project 'Piper' configuration respecting defaults and parameters.", PreRun: func(cmd *cobra.Command, args []string) { path, _ := os.Getwd() fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) initStageName(false) GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) }, Run: func(cmd *cobra.Command, _ []string) { if err := generateConfigWrapper(); err != nil { log.SetErrorCategory(log.ErrorConfiguration) log.Entry().WithError(err).Fatal("failed to retrieve configuration") } }, } addConfigFlags(createConfigCmd) return createConfigCmd } // GetDockerImageValue provides Piper commands additional access to configuration of step execution image if required func GetDockerImageValue(stepName string) (string, error) { configOptions.ContextConfig = true configOptions.StepName = stepName stepConfig, err := getConfig() if err != nil { return "", err } var dockerImageValue string dockerImageValue, ok := stepConfig.Config["dockerImage"].(string) if !ok { log.Entry().Infof("Config value of %v to compare with is not a string", stepConfig.Config["dockerImage"]) } return dockerImageValue, nil } func getBuildToolFromStageConfig(stepName string) (string, error) { configOptions.ContextConfig = true configOptions.StepName = stepName stageConfig, err := GetStageConfig() if err != nil { return "", err } buildTool, ok := stageConfig.Config["buildTool"].(string) if !ok { log.Entry().Infof("Config value of %v to compare with is not a string", stageConfig.Config["buildTool"]) } return buildTool, nil } // GetStageConfig provides Piper commands additional access to stage configuration if required. // This allows steps to refer to configuration parameters which are not part of the step itself. func GetStageConfig() (config.StepConfig, error) { myConfig := config.Config{} stepConfig := config.StepConfig{} projectConfigFile := getProjectConfigFile(GeneralConfig.CustomConfig) customConfig, err := configOptions.OpenFile(projectConfigFile, GeneralConfig.GitHubAccessTokens) if err != nil { if !errors.Is(err, os.ErrNotExist) { return stepConfig, errors.Wrapf(err, "config: open configuration file '%v' failed", projectConfigFile) } customConfig = nil } defaultConfig := []io.ReadCloser{} for _, f := range GeneralConfig.DefaultConfig { fc, err := configOptions.OpenFile(f, GeneralConfig.GitHubAccessTokens) // only create error for non-default values if err != nil && f != ".pipeline/defaults.yaml" { return stepConfig, errors.Wrapf(err, "config: getting defaults failed: '%v'", f) } if err == nil { defaultConfig = append(defaultConfig, fc) } } return myConfig.GetStageConfig(GeneralConfig.ParametersJSON, customConfig, defaultConfig, GeneralConfig.IgnoreCustomDefaults, configOptions.StageConfigAcceptedParameters, GeneralConfig.StageName) } func getConfig() (config.StepConfig, error) { var myConfig config.Config var stepConfig config.StepConfig var err error if configOptions.StageConfig { stepConfig, err = GetStageConfig() if err != nil { return stepConfig, errors.Wrap(err, "getting stage config failed") } } else { log.Entry().Infof("Printing stepName %s", configOptions.StepName) if GeneralConfig.MetaDataResolver == nil { GeneralConfig.MetaDataResolver = GetAllStepMetadata } metadata, err := config.ResolveMetadata(GeneralConfig.GitHubAccessTokens, GeneralConfig.MetaDataResolver, configOptions.StepMetadata, configOptions.StepName) if err != nil { return stepConfig, errors.Wrapf(err, "failed to resolve metadata") } // prepare output resource directories: // this is needed in order to have proper directory permissions in case // resources written inside a container image with a different user // Remark: This is so far only relevant for Jenkins environments where getConfig is executed prepareOutputEnvironment(metadata.Spec.Outputs.Resources, GeneralConfig.EnvRootPath) envParams := metadata.GetResourceParameters(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") reportingEnvParams := config.ReportingParameters.GetResourceParameters(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") resourceParams := mergeResourceParameters(envParams, reportingEnvParams) projectConfigFile := getProjectConfigFile(GeneralConfig.CustomConfig) customConfig, err := configOptions.OpenFile(projectConfigFile, GeneralConfig.GitHubAccessTokens) if err != nil { if !errors.Is(err, os.ErrNotExist) { return stepConfig, errors.Wrapf(err, "config: open configuration file '%v' failed", projectConfigFile) } customConfig = nil } defaultConfig, paramFilter, err := defaultsAndFilters(&metadata, metadata.Metadata.Name) if err != nil { return stepConfig, errors.Wrap(err, "defaults: retrieving step defaults failed") } for _, f := range GeneralConfig.DefaultConfig { fc, err := configOptions.OpenFile(f, GeneralConfig.GitHubAccessTokens) // only create error for non-default values if err != nil && f != ".pipeline/defaults.yaml" { return stepConfig, errors.Wrapf(err, "config: getting defaults failed: '%v'", f) } if err == nil { defaultConfig = append(defaultConfig, fc) } } var flags map[string]interface{} if configOptions.ContextConfig { metadata.Spec.Inputs.Parameters = []config.StepParameters{} } stepConfig, err = myConfig.GetStepConfig(flags, GeneralConfig.ParametersJSON, customConfig, defaultConfig, GeneralConfig.IgnoreCustomDefaults, paramFilter, metadata, resourceParams, GeneralConfig.StageName, metadata.Metadata.Name) if err != nil { return stepConfig, errors.Wrap(err, "getting step config failed") } // apply context conditions if context configuration is requested if configOptions.ContextConfig { applyContextConditions(metadata, &stepConfig) } } return stepConfig, nil } func generateConfigWrapper() error { var formatter func(interface{}) (string, error) switch strings.ToLower(configOptions.Output) { case "yaml", "yml": formatter = config.GetYAML case "json": formatter = config.GetJSON default: formatter = config.GetJSON } return GenerateConfig(formatter) } func GenerateConfig(formatter func(interface{}) (string, error)) error { utils := newGetConfigUtilsUtils() stepConfig, err := getConfig() if err != nil { return err } myConfig, err := formatter(stepConfig.Config) if err != nil { return fmt.Errorf("failed to marshal config: %w", err) } if len(configOptions.OutputFile) > 0 { if err := utils.FileWrite(configOptions.OutputFile, []byte(myConfig), 0666); err != nil { return fmt.Errorf("failed to write output file %v: %w", configOptions.OutputFile, err) } return nil } fmt.Println(myConfig) return nil } func addConfigFlags(cmd *cobra.Command) { // ToDo: support more output options, like https://kubernetes.io/docs/reference/kubectl/overview/#formatting-output cmd.Flags().StringVar(&configOptions.Output, "output", "json", "Defines the output format") cmd.Flags().StringVar(&configOptions.OutputFile, "outputFile", "", "Defines a file path. f set, the output will be written to the defines file") cmd.Flags().StringVar(&configOptions.ParametersJSON, "parametersJSON", os.Getenv("PIPER_parametersJSON"), "Parameters to be considered in JSON format") cmd.Flags().BoolVar(&configOptions.StageConfig, "stageConfig", false, "Defines if step stage configuration should be loaded and no step-specific config") cmd.Flags().StringArrayVar(&configOptions.StageConfigAcceptedParameters, "stageConfigAcceptedParams", []string{}, "Defines the parameters used for filtering stage/general configuration when accessing stage config") cmd.Flags().StringVar(&configOptions.StepMetadata, "stepMetadata", "", "Step metadata, passed as path to yaml") cmd.Flags().StringVar(&configOptions.StepName, "stepName", "", "Step name, used to get step metadata if yaml path is not set") cmd.Flags().BoolVar(&configOptions.ContextConfig, "contextConfig", false, "Defines if step context configuration should be loaded instead of step config") } func defaultsAndFilters(metadata *config.StepData, stepName string) ([]io.ReadCloser, config.StepFilters, error) { if configOptions.ContextConfig { defaults, err := metadata.GetContextDefaults(stepName) if err != nil { return nil, config.StepFilters{}, errors.Wrap(err, "metadata: getting context defaults failed") } return []io.ReadCloser{defaults}, metadata.GetContextParameterFilters(), nil } // ToDo: retrieve default values from metadata return []io.ReadCloser{}, metadata.GetParameterFilters(), nil } func applyContextConditions(metadata config.StepData, stepConfig *config.StepConfig) { // consider conditions for context configuration // containers config.ApplyContainerConditions(metadata.Spec.Containers, stepConfig) // sidecars config.ApplyContainerConditions(metadata.Spec.Sidecars, stepConfig) // ToDo: remove all unnecessary sub maps? // e.g. extract delete() from applyContainerConditions - loop over all stepConfig.Config[param.Value] and remove ... } func prepareOutputEnvironment(outputResources []config.StepResources, envRootPath string) { for _, oResource := range outputResources { for _, oParam := range oResource.Parameters { paramPath := path.Join(envRootPath, oResource.Name, fmt.Sprint(oParam["name"])) if oParam["fields"] != nil { paramFields, ok := oParam["fields"].([]map[string]string) if ok && len(paramFields) > 0 { paramPath = path.Join(paramPath, paramFields[0]["name"]) } } if _, err := os.Stat(filepath.Dir(paramPath)); errors.Is(err, os.ErrNotExist) { log.Entry().Debugf("Creating directory: %v", filepath.Dir(paramPath)) _ = os.MkdirAll(filepath.Dir(paramPath), 0777) } } } // prepare additional output directories known to possibly create permission issues when created from within a container // ToDo: evaluate if we can rather call this only in the correct step context (we know the step when calling getConfig!) // Could this be part of the container definition in the step.yaml? stepOutputDirectories := []string{ reporting.StepReportDirectory, // standard directory to collect md reports for pipelineCreateScanSummary ws.ReportsDirectory, // standard directory for reports created by whitesourceExecuteScan } for _, dir := range stepOutputDirectories { if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) { log.Entry().Debugf("Creating directory: %v", dir) _ = os.MkdirAll(dir, 0777) } } }