1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2024-12-12 10:55:20 +02:00

feat(checkIfStepActive): support new CRD style conditions (#3254)

* feat: first parts of new run struct

* add parts for new stage condition handling

* update conditions

* feat: finalize conditions and tests

* feat(checkIfStepActive): support new CRD style conditions

* feat(docs): allow generating stage docs

* chore(docs): make step directory configurable

* fix: tests

* add option to output file

* Update checkIfStepActive_test.go
This commit is contained in:
Oliver Nocon 2021-11-18 07:24:00 +01:00 committed by GitHub
parent 359cf9eeb3
commit 6c5434f957
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 904 additions and 30 deletions

View File

@ -1,11 +1,14 @@
package cmd
import (
"encoding/json"
"fmt"
"io"
"os"
"github.com/SAP/jenkins-library/pkg/config"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/bmatcuk/doublestar"
"github.com/pkg/errors"
"github.com/spf13/cobra"
@ -16,6 +19,9 @@ type checkStepActiveCommandOptions struct {
stageConfigFile string
stepName string
stageName string
v1Active bool
stageOutputFile string
stepOutputFile string
}
var checkStepActiveOptions checkStepActiveCommandOptions
@ -35,7 +41,8 @@ func CheckStepActiveCommand() *cobra.Command {
GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens)
},
Run: func(cmd *cobra.Command, _ []string) {
err := checkIfStepActive()
utils := &piperutils.Files{}
err := checkIfStepActive(utils)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
log.Entry().WithError(err).Fatal("Checking for an active step failed")
@ -46,7 +53,7 @@ func CheckStepActiveCommand() *cobra.Command {
return checkStepActiveCmd
}
func checkIfStepActive() error {
func checkIfStepActive(utils piperutils.FileUtils) error {
var pConfig config.Config
// load project config and defaults
@ -61,16 +68,62 @@ func checkIfStepActive() error {
}
defer stageConfigFile.Close()
runSteps := map[string]map[string]bool{}
runStages := map[string]bool{}
// load and evaluate step conditions
stageConditions := &config.RunConfig{StageConfigFile: stageConfigFile}
err = stageConditions.InitRunConfig(projectConfig, nil, nil, nil, nil, doublestar.Glob, checkStepActiveOptions.openFile)
if err != nil {
return err
if checkStepActiveOptions.v1Active {
runConfig := config.RunConfig{StageConfigFile: stageConfigFile}
runConfigV1 := &config.RunConfigV1{RunConfig: runConfig}
err = runConfigV1.InitRunConfigV1(projectConfig, nil, nil, nil, nil, utils)
if err != nil {
return err
}
runSteps = runConfigV1.RunSteps
runStages = runConfigV1.RunStages
} else {
runConfig := &config.RunConfig{StageConfigFile: stageConfigFile}
err = runConfig.InitRunConfig(projectConfig, nil, nil, nil, nil, doublestar.Glob, checkStepActiveOptions.openFile)
if err != nil {
return err
}
runSteps = runConfig.RunSteps
runStages = runConfig.RunStages
}
log.Entry().Debugf("RunSteps: %v", stageConditions.RunSteps)
log.Entry().Debugf("RunSteps: %v", runSteps)
log.Entry().Debugf("RunStages: %v", runStages)
if !stageConditions.RunSteps[checkStepActiveOptions.stageName][checkStepActiveOptions.stepName] {
if len(checkStepActiveOptions.stageOutputFile) > 0 || len(checkStepActiveOptions.stepOutputFile) > 0 {
if len(checkStepActiveOptions.stageOutputFile) > 0 {
result, err := json.Marshal(runStages)
if err != nil {
return fmt.Errorf("error marshalling json: %w", err)
}
log.Entry().Infof("Writing stage condition file %v", checkStepActiveOptions.stageOutputFile)
err = utils.FileWrite(checkStepActiveOptions.stageOutputFile, result, 0666)
if err != nil {
return fmt.Errorf("error writing file '%v': %w", checkStepActiveOptions.stageOutputFile, err)
}
}
if len(checkStepActiveOptions.stepOutputFile) > 0 {
result, err := json.Marshal(runSteps)
if err != nil {
return fmt.Errorf("error marshalling json: %w", err)
}
log.Entry().Infof("Writing step condition file %v", checkStepActiveOptions.stepOutputFile)
err = utils.FileWrite(checkStepActiveOptions.stepOutputFile, result, 0666)
if err != nil {
return fmt.Errorf("error writing file '%v': %w", checkStepActiveOptions.stepOutputFile, err)
}
}
// do not perform a check if output files are written
return nil
}
if !runSteps[checkStepActiveOptions.stageName][checkStepActiveOptions.stepName] {
return errors.Errorf("Step %s in stage %s is not active", checkStepActiveOptions.stepName, checkStepActiveOptions.stageName)
}
log.Entry().Infof("Step %s in stage %s is active", checkStepActiveOptions.stepName, checkStepActiveOptions.stageName)
@ -82,7 +135,10 @@ func addCheckStepActiveFlags(cmd *cobra.Command) {
cmd.Flags().StringVar(&checkStepActiveOptions.stageConfigFile, "stageConfig", ".resources/piper-stage-config.yml",
"Default config of piper pipeline stages")
cmd.Flags().StringVar(&checkStepActiveOptions.stepName, "step", "", "Name of the step being checked")
cmd.Flags().StringVar(&checkStepActiveOptions.stageName, "stage", "", "Name of the stage in which the step being checked is")
cmd.Flags().StringVar(&checkStepActiveOptions.stageName, "stage", "", "Name of the stage in which contains the step being checked")
cmd.Flags().BoolVar(&checkStepActiveOptions.v1Active, "useV1", false, "Use new CRD-style stage configuration")
cmd.Flags().StringVar(&checkStepActiveOptions.stageOutputFile, "stageOutputFile", "", "Defines a file path. If set, the stage output will be written to the defined file")
cmd.Flags().StringVar(&checkStepActiveOptions.stepOutputFile, "stepOutputFile", "", "Defines a file path. If set, the step output will be written to the defined file")
cmd.MarkFlagRequired("step")
cmd.MarkFlagRequired("stage")
}

View File

@ -58,7 +58,7 @@ func TestCheckStepActiveCommand(t *testing.T) {
})
t.Run("Optional flags", func(t *testing.T) {
exp := []string{"stageConfig"}
exp := []string{"stageConfig", "stageOutputFile", "stepOutputFile", "useV1"}
assert.Equal(t, exp, gotOpt, "optional flags incorrect")
})

View File

@ -2,10 +2,14 @@ package config
import (
"encoding/json"
"fmt"
"io"
"path"
"strings"
"github.com/SAP/jenkins-library/pkg/orchestrator"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/pkg/errors"
)
@ -17,6 +21,137 @@ const (
npmScriptsCondition = "npmScripts"
)
// EvaluateConditionsV1 validates stage conditions and updates runSteps in runConfig according to V1 schema
func (r *RunConfigV1) evaluateConditionsV1(config *Config, filters map[string]StepFilters, parameters map[string][]StepParameters,
secrets map[string][]StepSecrets, stepAliases map[string][]Alias, utils piperutils.FileUtils) error {
// initialize in case not initialized
if r.RunConfig.RunSteps == nil {
r.RunConfig.RunSteps = map[string]map[string]bool{}
}
if r.RunConfig.RunStages == nil {
r.RunConfig.RunStages = map[string]bool{}
}
for _, stage := range r.PipelineConfig.Spec.Stages {
runStep := map[string]bool{}
stageActive := false
// currently displayName is used, may need to consider to use technical name as well
stageName := stage.DisplayName
for _, step := range stage.Steps {
// Only consider orchestrator-specific steps in case orchestrator limitation is set
currentOrchestrator := orchestrator.DetectOrchestrator().String()
if len(step.Orchestrators) > 0 && !piperutils.ContainsString(step.Orchestrators, currentOrchestrator) {
continue
}
stepActive := false
stepConfig, err := r.getStepConfig(config, stageName, step.Name, filters, parameters, secrets, stepAliases)
if err != nil {
return err
}
if active, ok := stepConfig.Config[step.Name].(bool); ok {
// respect explicit activation/de-activation if available
stepActive = active
} else {
if step.Conditions == nil || len(step.Conditions) == 0 {
// if no condition is available, step will be active by default
stepActive = true
} else {
for _, condition := range step.Conditions {
stepActive, err = condition.evaluateV1(stepConfig, utils)
if err != nil {
return fmt.Errorf("failed to evaluate stage conditions: %w", err)
}
if stepActive {
// first condition which matches will be considered to activate the step
break
}
}
}
}
if stepActive {
stageActive = true
}
runStep[step.Name] = stepActive
r.RunSteps[stageName] = runStep
}
r.RunStages[stageName] = stageActive
}
return nil
}
func (s *StepCondition) evaluateV1(config StepConfig, utils piperutils.FileUtils) (bool, error) {
// only the first condition will be evaluated.
// if multiple conditions should be checked they need to provided via the Conditions list
if s.Config != nil {
if len(s.Config) > 1 {
return false, errors.Errorf("only one config key allowed per condition but %v provided", len(s.Config))
}
// for loop will only cover first entry since we throw an error in case there is more than one config key defined already above
for param, activationValues := range s.Config {
for _, activationValue := range activationValues {
if activationValue == config.Config[param] {
return true, nil
}
}
return false, nil
}
}
if len(s.ConfigKey) > 0 {
if configValue := config.Config[s.ConfigKey]; configValue != nil {
return true, nil
}
return false, nil
}
if len(s.FilePattern) > 0 {
files, err := utils.Glob(s.FilePattern)
if err != nil {
return false, errors.Wrap(err, "failed to check filePattern condition")
}
if len(files) > 0 {
return true, nil
}
return false, nil
}
if len(s.FilePatternFromConfig) > 0 {
configValue := fmt.Sprint(config.Config[s.FilePatternFromConfig])
if len(configValue) == 0 {
return false, nil
}
files, err := utils.Glob(configValue)
if err != nil {
return false, errors.Wrap(err, "failed to check filePatternFromConfig condition")
}
if len(files) > 0 {
return true, nil
}
return false, nil
}
if len(s.NpmScript) > 0 {
return checkForNpmScriptsInPackagesV1(s.NpmScript, config, utils)
}
// needs to be checked last:
// if none of the other conditions matches, step will be active unless set to inactive
if s.Inactive == true {
return false, nil
} else {
return true, nil
}
}
// EvaluateConditions validates stage conditions and updates runSteps in runConfig
func (r *RunConfig) evaluateConditions(config *Config, filters map[string]StepFilters, parameters map[string][]StepParameters,
secrets map[string][]StepSecrets, stepAliases map[string][]Alias, glob func(pattern string) (matches []string, err error)) error {
@ -108,6 +243,13 @@ func checkConfig(condition interface{}, config StepConfig, stepName string) (boo
return false, nil
}
func checkConfigKey(configKey string, config StepConfig, stepName string) (bool, error) {
if configValue := stepConfigLookup(config.Config, stepName, configKey); configValue != nil {
return true, nil
}
return false, nil
}
func checkConfigKeys(condition interface{}, config StepConfig, stepName string) (bool, error) {
arrCondition, ok := condition.([]interface{})
if !ok {
@ -235,3 +377,44 @@ func checkForNpmScriptsInPackages(condition interface{}, config StepConfig, step
}
return false, nil
}
func checkForNpmScriptsInPackagesV1(npmScript string, config StepConfig, utils piperutils.FileUtils) (bool, error) {
packages, err := utils.Glob("**/package.json")
if err != nil {
return false, errors.Wrap(err, "failed to check if file-exists")
}
for _, pack := range packages {
packDirs := strings.Split(path.Dir(pack), "/")
isNodeModules := false
for _, dir := range packDirs {
if dir == "node_modules" {
isNodeModules = true
break
}
}
if isNodeModules {
continue
}
jsonFile, err := utils.FileRead(pack)
if err != nil {
return false, errors.Errorf("failed to open file %s: %v", pack, err)
}
packageJSON := map[string]interface{}{}
if err := json.Unmarshal(jsonFile, &packageJSON); err != nil {
return false, errors.Errorf("failed to unmarshal json file %s: %v", pack, err)
}
npmScripts, ok := packageJSON["scripts"]
if !ok {
continue
}
scriptsMap, ok := npmScripts.(map[string]interface{})
if !ok {
return false, errors.Errorf("failed to read scripts from package.json: %T", npmScripts)
}
if _, ok := scriptsMap[npmScript]; ok {
return true, nil
}
}
return false, nil
}

View File

@ -2,11 +2,13 @@ package config
import (
"errors"
"fmt"
"io"
"io/ioutil"
"strings"
"testing"
"github.com/SAP/jenkins-library/pkg/mock"
"github.com/stretchr/testify/assert"
)
@ -42,7 +44,245 @@ func evaluateConditionsOpenFileMock(name string, _ map[string]string) (io.ReadCl
return fileContent, nil
}
func Test_evaluateConditions(t *testing.T) {
func TestEvaluateConditionsV1(t *testing.T) {
filesMock := mock.FilesMock{}
runConfig := RunConfigV1{
PipelineConfig: PipelineDefinitionV1{
Spec: Spec{
Stages: []Stage{
{
Name: "stage1",
DisplayName: "Test Stage 1",
Steps: []Step{
{
Name: "step1_1",
Conditions: []StepCondition{},
Orchestrators: []string{"Jenkins"},
},
{
Name: "step1_2",
Conditions: []StepCondition{
{ConfigKey: "testKey"},
},
},
{
Name: "step1_3",
Conditions: []StepCondition{},
},
},
},
{
Name: "stage2",
DisplayName: "Test Stage 2",
Steps: []Step{
{
Name: "step2_1",
Conditions: []StepCondition{
{ConfigKey: "testKeyNotExisting"},
{ConfigKey: "testKey"},
},
},
{
Name: "step2_2",
},
},
},
{
Name: "stage3",
DisplayName: "Test Stage 3",
Steps: []Step{
{
Name: "step3_1",
Conditions: []StepCondition{
{ConfigKey: "testKeyNotExisting"},
{ConfigKey: "testKey"},
},
},
},
},
},
},
},
}
config := Config{Stages: map[string]map[string]interface{}{
"Test Stage 1": {"step1_3": false, "testKey": "testVal"},
"Test Stage 2": {"testKey": "testVal"},
}}
expectedSteps := map[string]map[string]bool{
"Test Stage 1": {
"step1_2": true,
"step1_3": false,
},
"Test Stage 2": {
"step2_1": true,
"step2_2": true,
},
"Test Stage 3": {
"step3_1": false,
},
}
expectedStages := map[string]bool{
"Test Stage 1": true,
"Test Stage 2": true,
"Test Stage 3": false,
}
err := runConfig.evaluateConditionsV1(&config, nil, nil, nil, nil, &filesMock)
assert.NoError(t, err)
assert.Equal(t, expectedSteps, runConfig.RunSteps)
assert.Equal(t, expectedStages, runConfig.RunStages)
}
func TestEvaluateV1(t *testing.T) {
tt := []struct {
name string
config StepConfig
stepCondition StepCondition
expected bool
expectedError error
}{
{
name: "Config condition - true",
config: StepConfig{Config: map[string]interface{}{
"deployTool": "helm3",
}},
stepCondition: StepCondition{Config: map[string][]interface{}{"deployTool": {"helm", "helm3", "kubectl"}}},
expected: true,
},
{
name: "Config condition - false",
config: StepConfig{Config: map[string]interface{}{
"deployTool": "notsupported",
}},
stepCondition: StepCondition{Config: map[string][]interface{}{"deployTool": {"helm", "helm3", "kubectl"}}},
expected: false,
},
{
name: "Config condition - integer - true",
config: StepConfig{Config: map[string]interface{}{
"executors": 1,
}},
stepCondition: StepCondition{Config: map[string][]interface{}{"executors": {1}}},
expected: true,
},
{
name: "Config condition - wrong condition definition",
config: StepConfig{Config: map[string]interface{}{
"deployTool": "helm3",
}},
stepCondition: StepCondition{Config: map[string][]interface{}{"deployTool": {"helm", "helm3", "kubectl"}, "deployTool2": {"myTool"}}},
expectedError: fmt.Errorf("only one config key allowed per condition but 2 provided"),
},
{
name: "ConfigKey condition - true",
config: StepConfig{Config: map[string]interface{}{
"dockerRegistryUrl": "https://my.docker.registry.url",
}},
stepCondition: StepCondition{ConfigKey: "dockerRegistryUrl"},
expected: true,
},
{
name: "ConfigKey condition - false",
config: StepConfig{Config: map[string]interface{}{}},
stepCondition: StepCondition{ConfigKey: "dockerRegistryUrl"},
expected: false,
},
{
name: "FilePattern condition - true",
config: StepConfig{Config: map[string]interface{}{}},
stepCondition: StepCondition{FilePattern: "**/conf.js"},
expected: true,
},
{
name: "FilePattern condition - false",
config: StepConfig{Config: map[string]interface{}{}},
stepCondition: StepCondition{FilePattern: "**/confx.js"},
expected: false,
},
{
name: "FilePatternFromConfig condition - true",
config: StepConfig{Config: map[string]interface{}{
"newmanCollection": "**/*.postman_collection.json",
}},
stepCondition: StepCondition{FilePatternFromConfig: "newmanCollection"},
expected: true,
},
{
name: "FilePatternFromConfig condition - false",
config: StepConfig{Config: map[string]interface{}{
"newmanCollection": "**/*.postmanx_collection.json",
}},
stepCondition: StepCondition{FilePatternFromConfig: "newmanCollection"},
expected: false,
},
{
name: "FilePatternFromConfig condition - false, empty value",
config: StepConfig{Config: map[string]interface{}{
"newmanCollection": "",
}},
stepCondition: StepCondition{FilePatternFromConfig: "newmanCollection"},
expected: false,
},
{
name: "NpmScript condition - true",
config: StepConfig{Config: map[string]interface{}{}},
stepCondition: StepCondition{NpmScript: "testScript"},
expected: true,
},
{
name: "NpmScript condition - true",
config: StepConfig{Config: map[string]interface{}{}},
stepCondition: StepCondition{NpmScript: "missingScript"},
expected: false,
},
{
name: "Inactive condition - false",
config: StepConfig{Config: map[string]interface{}{}},
stepCondition: StepCondition{Inactive: true},
expected: false,
},
{
name: "Inactive condition - true",
config: StepConfig{Config: map[string]interface{}{}},
stepCondition: StepCondition{Inactive: false},
expected: true,
},
{
name: "No condition - true",
config: StepConfig{Config: map[string]interface{}{}},
expected: true,
},
}
packageJson := `{
"scripts": {
"testScript": "whatever"
}
}`
filesMock := mock.FilesMock{}
filesMock.AddFile("conf.js", []byte("//test"))
filesMock.AddFile("my.postman_collection.json", []byte("{}"))
filesMock.AddFile("package.json", []byte(packageJson))
for _, test := range tt {
t.Run(test.name, func(t *testing.T) {
active, err := test.stepCondition.evaluateV1(test.config, &filesMock)
if test.expectedError == nil {
assert.NoError(t, err)
} else {
assert.EqualError(t, err, fmt.Sprint(test.expectedError))
}
assert.Equal(t, test.expected, active)
})
}
}
func TestEvaluateConditions(t *testing.T) {
tests := []struct {
name string
customConfig *Config

View File

@ -1,9 +1,11 @@
package config
import (
"fmt"
"io"
"io/ioutil"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/ghodss/yaml"
"github.com/pkg/errors"
)
@ -12,8 +14,15 @@ import (
type RunConfig struct {
StageConfigFile io.ReadCloser
StageConfig StageConfig
RunStages map[string]bool
RunSteps map[string]map[string]bool
OpenFile func(s string, t map[string]string) (io.ReadCloser, error)
FileUtils *piperutils.Files
}
type RunConfigV1 struct {
RunConfig
PipelineConfig PipelineDefinitionV1
}
type StageConfig struct {
@ -24,6 +33,65 @@ type StepConditions struct {
Conditions map[string]map[string]interface{} `json:"stepConditions,omitempty"`
}
type PipelineDefinitionV1 struct {
APIVersion string `json:"apiVersion"`
Kind string `json:"kind"`
Metadata Metadata `json:"metadata"`
Spec Spec `json:"spec"`
openFile func(s string, t map[string]string) (io.ReadCloser, error)
runSteps map[string]map[string]bool
}
type Metadata struct {
Name string `json:"name,omitempty"`
DisplayName string `json:"displayName,omitempty"`
Description string `json:"description,omitempty"`
}
type Spec struct {
Stages []Stage `json:"stages"`
}
type Stage struct {
Name string `json:"name,omitempty"`
DisplayName string `json:"displayName,omitempty"`
Description string `json:"description,omitempty"`
Steps []Step `json:"steps,omitempty"`
}
type Step struct {
Name string `json:"name,omitempty"`
Description string `json:"description,omitempty"`
Conditions []StepCondition `json:"conditions,omitempty"`
Orchestrators []string `json:"orchestrators,omitempty"`
}
type StepCondition struct {
Config map[string][]interface{} `json:"config,omitempty"`
ConfigKey string `json:"configKey,omitempty"`
FilePattern string `json:"filePattern,omitempty"`
FilePatternFromConfig string `json:"filePatternFromConfig,omitempty"`
Inactive bool `json:"inactive,omitempty"`
NpmScript string `json:"npmScript,omitempty"`
}
func (r *RunConfigV1) InitRunConfigV1(config *Config, filters map[string]StepFilters, parameters map[string][]StepParameters,
secrets map[string][]StepSecrets, stepAliases map[string][]Alias, utils piperutils.FileUtils) error {
if len(r.PipelineConfig.Spec.Stages) == 0 {
if err := r.loadConditionsV1(); err != nil {
return fmt.Errorf("failed to load pipeline run conditions: %w", err)
}
}
err := r.evaluateConditionsV1(config, filters, parameters, secrets, stepAliases, utils)
if err != nil {
return fmt.Errorf("failed to evaluate step conditions: %w", err)
}
return nil
}
// InitRunConfig ...
func (r *RunConfig) InitRunConfig(config *Config, filters map[string]StepFilters, parameters map[string][]StepParameters,
secrets map[string][]StepSecrets, stepAliases map[string][]Alias, glob func(pattern string) (matches []string, err error),
@ -77,6 +145,20 @@ func (r *RunConfig) loadConditions() error {
return nil
}
func (r *RunConfigV1) loadConditionsV1() error {
defer r.StageConfigFile.Close()
content, err := ioutil.ReadAll(r.StageConfigFile)
if err != nil {
return errors.Wrapf(err, "error: failed to read the stageConfig file")
}
err = yaml.Unmarshal(content, &r.PipelineConfig)
if err != nil {
return errors.Errorf("format of configuration is invalid %q: %v", content, err)
}
return nil
}
func stepConfigLookup(m map[string]interface{}, stepName, key string) interface{} {
// flat map: key is on top level
if m[key] != nil {

View File

@ -1,12 +1,14 @@
package config
import (
"fmt"
"io"
"io/ioutil"
"reflect"
"strings"
"testing"
"github.com/SAP/jenkins-library/pkg/mock"
"github.com/stretchr/testify/assert"
)
@ -21,6 +23,53 @@ func initRunConfigGlobMock(pattern string) ([]string, error) {
return matches, nil
}
func TestInitRunConfigV1(t *testing.T) {
tt := []struct {
name string
config Config
stageConfig string
runStagesExpected map[string]bool
runStepsExpected map[string]map[string]bool
expectedError error
errorContains string
}{
{
name: "success",
config: Config{Stages: map[string]map[string]interface{}{"testStage": {"testKey": "testVal"}}},
stageConfig: "spec:\n stages:\n - name: testStage\n displayName: testStage\n steps:\n - name: testStep\n conditions:\n - configKey: testKey",
runStepsExpected: map[string]map[string]bool{},
},
{
name: "error - load conditions",
stageConfig: "wrong stage config format",
runStepsExpected: map[string]map[string]bool{},
errorContains: "failed to load pipeline run conditions",
},
{
name: "error - evaluate conditions",
config: Config{Stages: map[string]map[string]interface{}{"testStage": {"testKey": "testVal"}}},
runStepsExpected: map[string]map[string]bool{},
stageConfig: "spec:\n stages:\n - name: testStage\n displayName: testStage\n steps:\n - name: testStep\n conditions:\n - config:\n configKey1:\n - configVal1\n configKey2:\n - configVal2",
errorContains: "failed to evaluate step conditions",
},
}
filesMock := mock.FilesMock{}
for _, test := range tt {
stageConfig := ioutil.NopCloser(strings.NewReader(test.stageConfig))
runConfig := RunConfig{StageConfigFile: stageConfig}
runConfigV1 := RunConfigV1{RunConfig: runConfig}
err := runConfigV1.InitRunConfigV1(&test.config, nil, nil, nil, nil, &filesMock)
if len(test.errorContains) > 0 {
assert.Contains(t, fmt.Sprint(err), test.errorContains)
} else {
assert.NoError(t, err)
}
}
}
func TestInitRunConfig(t *testing.T) {
tests := []struct {
name string

View File

@ -10,6 +10,7 @@ import (
generator "github.com/SAP/jenkins-library/pkg/documentation/generator"
"github.com/SAP/jenkins-library/pkg/generator/helper"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/ghodss/yaml"
)
@ -27,42 +28,67 @@ func (f *sliceFlags) Set(value string) error {
}
func main() {
// flags for step documentation
var metadataPath string
var docTemplatePath string
var customLibraryStepFile string
var customDefaultFiles sliceFlags
var includeAzure bool
flag.StringVar(&metadataPath, "metadataDir", "./resources/metadata", "The directory containing the step metadata. Default points to \\'resources/metadata\\'.")
flag.StringVar(&docTemplatePath, "docuDir", "./documentation/docs/steps/", "The directory containing the docu stubs. Default points to \\'documentation/docs/steps/\\'.")
flag.StringVar(&customLibraryStepFile, "customLibraryStepFile", "", "")
flag.Var(&customDefaultFiles, "customDefaultFile", "Path to a custom default configuration file.")
flag.BoolVar(&includeAzure, "includeAzure", false, "Include Azure-specifics in step documentation.")
// flags for stage documentation
var generateStageConfig bool
var stageMetadataPath string
var stageTargetPath string
var relativeStepsPath string
flag.BoolVar(&generateStageConfig, "generateStageConfig", false, "Create stage documentation instead of step documentation.")
flag.StringVar(&stageMetadataPath, "stageMetadataPath", "./resources/com.sap.piper/pipeline/stageDefaults.yml", "The file containing the stage metadata. Default points to \\'./resources/com.sap.piper/pipeline/stageDefaults.yml\\'.")
flag.StringVar(&stageTargetPath, "stageTargetPath", "./documentation/docs/stages/", "The target path for the generated stage documentation. Default points to \\'./documentation/docs/stages/\\'.")
flag.StringVar(&relativeStepsPath, "relativeStepsPath", "../../steps", "The relative path from stages to steps")
flag.Parse()
fmt.Println("using Metadata Directory:", metadataPath)
fmt.Println("using Documentation Directory:", docTemplatePath)
fmt.Println("using Custom Default Files:", strings.Join(customDefaultFiles.list, ", "))
if generateStageConfig {
// generating stage documentation
fmt.Println("Generating STAGE documentation")
fmt.Println("using Metadata:", stageMetadataPath)
fmt.Println("using stage target directory:", stageTargetPath)
fmt.Println("using relative steps path:", relativeStepsPath)
if len(customLibraryStepFile) > 0 {
fmt.Println("Reading custom library step mapping..")
content, err := ioutil.ReadFile(customLibraryStepFile)
utils := &piperutils.Files{}
err := generator.GenerateStageDocumentation(stageMetadataPath, stageTargetPath, relativeStepsPath, utils)
checkError(err)
err = yaml.Unmarshal(content, &generator.CustomLibrarySteps)
} else {
// generating step documentation
fmt.Println("Generating STEP documentation")
fmt.Println("using Metadata Directory:", metadataPath)
fmt.Println("using Documentation Directory:", docTemplatePath)
fmt.Println("using Custom Default Files:", strings.Join(customDefaultFiles.list, ", "))
if len(customLibraryStepFile) > 0 {
fmt.Println("Reading custom library step mapping..")
content, err := ioutil.ReadFile(customLibraryStepFile)
checkError(err)
err = yaml.Unmarshal(content, &generator.CustomLibrarySteps)
checkError(err)
fmt.Println(generator.CustomLibrarySteps)
}
metadataFiles, err := helper.MetadataFiles(metadataPath)
checkError(err)
err = generator.GenerateStepDocumentation(metadataFiles, customDefaultFiles.list, generator.DocuHelperData{
DocTemplatePath: docTemplatePath,
OpenDocTemplateFile: openDocTemplateFile,
DocFileWriter: writeFile,
OpenFile: openFile,
}, includeAzure)
checkError(err)
fmt.Println(generator.CustomLibrarySteps)
}
metadataFiles, err := helper.MetadataFiles(metadataPath)
checkError(err)
err = generator.GenerateStepDocumentation(metadataFiles, customDefaultFiles.list, generator.DocuHelperData{
DocTemplatePath: docTemplatePath,
OpenDocTemplateFile: openDocTemplateFile,
DocFileWriter: writeFile,
OpenFile: openFile,
}, includeAzure)
checkError(err)
}
func openDocTemplateFile(docTemplateFilePath string) (io.ReadCloser, error) {

View File

@ -4,10 +4,16 @@ import (
"bytes"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strings"
"text/template"
"github.com/SAP/jenkins-library/pkg/config"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/ghodss/yaml"
)
// DocuHelperData is used to transport the needed parameters and functions from the step generator to the docu generation.
@ -225,3 +231,194 @@ func appendGeneralOptionsToParameters(stepData *config.StepData) {
}
stepData.Spec.Inputs.Parameters = append(stepData.Spec.Inputs.Parameters, script, verbose)
}
// GenerateStepDocumentation generates pipeline stage documentation based on pipeline configuration provided in a yaml file
func GenerateStageDocumentation(stageMetadataPath, stageTargetPath, relativeStepsPath string, utils piperutils.FileUtils) error {
if len(stageTargetPath) == 0 {
return fmt.Errorf("stageTargetPath cannot be empty")
}
if len(stageMetadataPath) == 0 {
return fmt.Errorf("stageMetadataPath cannot be empty")
}
if err := utils.MkdirAll(stageTargetPath, 0777); err != nil {
return fmt.Errorf("failed to create directory '%v': %w", stageTargetPath, err)
}
stageMetadataContent, err := utils.FileRead(stageMetadataPath)
if err != nil {
return fmt.Errorf("failed to read stage metadata file '%v': %w", stageMetadataPath, err)
}
stageRunConfig := config.RunConfigV1{}
err = yaml.Unmarshal(stageMetadataContent, &stageRunConfig.PipelineConfig)
if err != nil {
return fmt.Errorf("format of configuration is invalid %q: %w", stageMetadataContent, err)
}
err = createPipelineDocumentation(&stageRunConfig, stageTargetPath, relativeStepsPath, utils)
if err != nil {
return fmt.Errorf("failed to create pipeline documentation: %w", err)
}
return nil
}
func createPipelineDocumentation(stageRunConfig *config.RunConfigV1, stageTargetPath, relativeStepsPath string, utils piperutils.FileUtils) error {
if err := createPipelineOverviewDocumentation(stageRunConfig, stageTargetPath, utils); err != nil {
return fmt.Errorf("failed to create pipeline overview: %w", err)
}
if err := createPipelineStageDocumentation(stageRunConfig, stageTargetPath, relativeStepsPath, utils); err != nil {
return fmt.Errorf("failed to create pipeline stage details: %w", err)
}
return nil
}
func createPipelineOverviewDocumentation(stageRunConfig *config.RunConfigV1, stageTargetPath string, utils piperutils.FileUtils) error {
overviewFileName := "overview.md"
overviewDoc := fmt.Sprintf("# %v\n\n", stageRunConfig.PipelineConfig.Metadata.DisplayName)
overviewDoc += fmt.Sprintf("%v\n\n", stageRunConfig.PipelineConfig.Metadata.Description)
overviewDoc += fmt.Sprintf("The %v comprises following stages\n\n", stageRunConfig.PipelineConfig.Metadata.Description)
for _, stage := range stageRunConfig.PipelineConfig.Spec.Stages {
stageFilePath := filepath.Join(stageTargetPath, fmt.Sprintf("%v.md", stage.Name))
overviewDoc += fmt.Sprintf("* [%v Stage](%v)", stage.DisplayName, stageFilePath)
}
overviewFilePath := filepath.Join(stageTargetPath, overviewFileName)
fmt.Println("writing file", overviewFilePath)
return utils.FileWrite(overviewFilePath, []byte(overviewDoc), 0666)
}
const stepConditionDetails = `!!! note "Step condition details"
There are currently several conditions which can be checked.<br />**Important: It will be sufficient that any one condition per step is met.**
* ` + "`" + `config key` + "`" + `: Checks if a defined configuration parameter is set.
* ` + "`" + `config value` + "`" + `: Checks if a configuration parameter has a defined value.
* ` + "`" + `file pattern` + "`" + `: Checks if files according a defined pattern exist in the project.
* ` + "`" + `file pattern from config` + "`" + `: Checks if files according a pattern defined in the custom configuration exist in the project.
* ` + "`" + `npm script` + "`" + `: Checks if a npm script exists in one of the package.json files in the repositories.
`
const overrulingStepActivation = `!!! note "Overruling step activation conditions"
It is possible to overrule the automatically detected step activation status.
* In case a step will be **active** you can add to your stage configuration ` + "`" + `<stepName>: false` + "`" + ` to explicitly **deactivate** the step.
* In case a step will be **inactive** you can add to your stage configuration ` + "`" + `<stepName>: true` + "`" + ` to explicitly **activate** the step.
`
func createPipelineStageDocumentation(stageRunConfig *config.RunConfigV1, stageTargetPath, relativeStepsPath string, utils piperutils.FileUtils) error {
for _, stage := range stageRunConfig.PipelineConfig.Spec.Stages {
stageDoc := fmt.Sprintf("# %v\n\n", stage.DisplayName)
stageDoc += fmt.Sprintf("%v\n\n", stage.Description)
stageDoc += "## Stage Content\n\nThis stage comprises following steps which are activated depending on your use-case/configuration:\n\n"
for i, step := range stage.Steps {
if i == 0 {
stageDoc += "| step | step description |\n"
stageDoc += "| ---- | ---------------- |\n"
}
orchestratorBadges := ""
for _, orchestrator := range step.Orchestrators {
orchestratorBadges += getBadge(orchestrator) + " "
}
stageDoc += fmt.Sprintf("| [%v](%v/%v.md) | %v%v |\n", step.Name, relativeStepsPath, step.Name, orchestratorBadges, step.Description)
}
stageDoc += "\n"
stageDoc += "## Stage & Step Activation\n\nThis stage will be active in case one of following conditions are met:\n\n"
stageDoc += "* One of the steps is explicitly activated by using `<stepName>: true` in the stage configuration\n"
stageDoc += "* At least one of the step conditions is met and steps are not explicitly deactivated by using `<stepName>: false` in the stage configuration\n\n"
stageDoc += stepConditionDetails
stageDoc += overrulingStepActivation
stageDoc += "Following conditions apply for activation of steps contained in the stage:\n\n"
stageDoc += "| step | active if one of following conditions is met |\n"
stageDoc += "| ---- | -------------------------------------------- |\n"
// add step condition details
for _, step := range stage.Steps {
stageDoc += fmt.Sprintf("| [%v](%v/%v.md) | %v |\n", step.Name, relativeStepsPath, step.Name, getStepConditionDetails(step))
}
stageFilePath := filepath.Join(stageTargetPath, fmt.Sprintf("%v.md", stage.Name))
fmt.Println("writing file", stageFilePath)
if err := utils.FileWrite(stageFilePath, []byte(stageDoc), 0666); err != nil {
return fmt.Errorf("failed to write stage file '%v': %w", stageFilePath, err)
}
}
return nil
}
func getBadge(orchestrator string) string {
orchestratorOnly := strings.Title(strings.ToLower(orchestrator)) + " only"
urlPath := &url.URL{Path: orchestratorOnly}
orchestratorOnlyString := urlPath.String()
return fmt.Sprintf("[![%v](https://img.shields.io/badge/-%v-yellowgreen)](#)", orchestratorOnly, orchestratorOnlyString)
}
func getStepConditionDetails(step config.Step) string {
stepConditions := ""
if step.Conditions == nil || len(step.Conditions) == 0 {
return "**active** by default - deactivate explicitly"
}
if len(step.Orchestrators) > 0 {
orchestratorBadges := ""
for _, orchestrator := range step.Orchestrators {
orchestratorBadges += getBadge(orchestrator) + " "
}
stepConditions = orchestratorBadges + "<br />"
}
for _, condition := range step.Conditions {
if condition.Config != nil && len(condition.Config) > 0 {
stepConditions += "<i>config:</i><ul>"
for param, activationValues := range condition.Config {
for _, activationValue := range activationValues {
stepConditions += fmt.Sprintf("<li>`%v`: `%v`</li>", param, activationValue)
}
// config condition only covers first entry
break
}
stepConditions += "</ul>"
continue
}
if len(condition.ConfigKey) > 0 {
stepConditions += fmt.Sprintf("<i>config key:</i><ul><li>`%v`</li></ul>", condition.ConfigKey)
continue
}
if len(condition.FilePattern) > 0 {
stepConditions += fmt.Sprintf("<i>file pattern:</i><ul><li>`%v`</li></ul>", condition.FilePattern)
continue
}
if len(condition.FilePatternFromConfig) > 0 {
stepConditions += fmt.Sprintf("<i>file pattern from config:</i><ul><li>`%v`</li></ul>", condition.FilePatternFromConfig)
continue
}
if len(condition.NpmScript) > 0 {
stepConditions += fmt.Sprintf("<i>npm script:</i><ul><li>`%v`</li></ul>", condition.NpmScript)
continue
}
if condition.Inactive {
stepConditions += "**inactive** by default - activate explicitly"
continue
}
}
return stepConditions
}

View File

@ -88,3 +88,44 @@ func TestSetDefaultAndPossisbleValues(t *testing.T) {
assert.Equal(t, []interface{}{true, false}, stepData.Spec.Inputs.Parameters[0].PossibleValues)
}
func TestGetBadge(t *testing.T) {
tt := []struct {
in string
expected string
}{
{in: "Jenkins", expected: "[![Jenkins only](https://img.shields.io/badge/-Jenkins%20only-yellowgreen)](#)"},
{in: "jenkins", expected: "[![Jenkins only](https://img.shields.io/badge/-Jenkins%20only-yellowgreen)](#)"},
{in: "Azure", expected: "[![Azure only](https://img.shields.io/badge/-Azure%20only-yellowgreen)](#)"},
{in: "azure", expected: "[![Azure only](https://img.shields.io/badge/-Azure%20only-yellowgreen)](#)"},
{in: "Github Actions", expected: "[![Github Actions only](https://img.shields.io/badge/-Github%20Actions%20only-yellowgreen)](#)"},
{in: "github actions", expected: "[![Github Actions only](https://img.shields.io/badge/-Github%20Actions%20only-yellowgreen)](#)"},
}
for _, test := range tt {
assert.Equal(t, test.expected, getBadge(test.in))
}
}
func TestGetStepConditionDetails(t *testing.T) {
tt := []struct {
name string
step config.Step
expected string
}{
{name: "noCondition", step: config.Step{Conditions: []config.StepCondition{}}, expected: "**active** by default - deactivate explicitly"},
{name: "config", step: config.Step{Conditions: []config.StepCondition{{Config: map[string][]interface{}{"configKey1": {"keyVal1", "keyVal2"}}}}}, expected: "<i>config:</i><ul><li>`configKey1`: `keyVal1`</li><li>`configKey1`: `keyVal2`</li></ul>"},
{name: "configKey", step: config.Step{Conditions: []config.StepCondition{{ConfigKey: "configKey"}}}, expected: "<i>config key:</i><ul><li>`configKey`</li></ul>"},
{name: "filePattern", step: config.Step{Conditions: []config.StepCondition{{FilePattern: "testPattern"}}}, expected: "<i>file pattern:</i><ul><li>`testPattern`</li></ul>"},
{name: "filePatternFromConfig", step: config.Step{Conditions: []config.StepCondition{{FilePatternFromConfig: "patternConfigKey"}}}, expected: "<i>file pattern from config:</i><ul><li>`patternConfigKey`</li></ul>"},
{name: "inactive", step: config.Step{Conditions: []config.StepCondition{{Inactive: true}}}, expected: "**inactive** by default - activate explicitly"},
{name: "npmScript", step: config.Step{Conditions: []config.StepCondition{{NpmScript: "testScript"}}}, expected: "<i>npm script:</i><ul><li>`testScript`</li></ul>"},
{name: "multiple conditions", step: config.Step{Conditions: []config.StepCondition{{ConfigKey: "configKey"}, {FilePattern: "testPattern"}}}, expected: "<i>config key:</i><ul><li>`configKey`</li></ul><i>file pattern:</i><ul><li>`testPattern`</li></ul>"},
}
for _, test := range tt {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, getStepConditionDetails(test.step))
})
}
}