1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-01-16 05:16:08 +02:00

Convert step/stage condition logic to golang (#2993)

* Added checkIfStepActive step

* Implemented npmScripts condition. Code was refactored

* Added some unit tests

* Fixed go modules

* Fixed go modules

Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
This commit is contained in:
Siarhei Pazdniakou 2021-07-26 08:47:43 +03:00 committed by GitHub
parent 39858cde2b
commit 7a325e6fc8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 1341 additions and 0 deletions

125
cmd/checkIfStepActive.go Normal file
View File

@ -0,0 +1,125 @@
package cmd
import (
"io"
"os"
"github.com/SAP/jenkins-library/pkg/config"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/bmatcuk/doublestar"
"github.com/pkg/errors"
"github.com/spf13/cobra"
)
type checkStepActiveCommandOptions struct {
openFile func(s string, t map[string]string) (io.ReadCloser, error)
stageConfigFile string
stepName string
stageName string
}
var checkStepActiveOptions checkStepActiveCommandOptions
// CheckStepActiveCommand is the entry command for checking if a step is active in a defined stage
func CheckStepActiveCommand() *cobra.Command {
checkStepActiveOptions.openFile = config.OpenPiperFile
var checkStepActiveCmd = &cobra.Command{
Use: "checkIfStepActive",
Short: "Checks if a step is active in a defined stage.",
PreRun: func(cmd *cobra.Command, args []string) {
path, _ := os.Getwd()
fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path}
log.RegisterHook(fatalHook)
initStageName(false)
log.SetVerbose(GeneralConfig.Verbose)
GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens)
},
Run: func(cmd *cobra.Command, _ []string) {
err := checkIfStepActive()
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
log.Entry().WithError(err).Fatal("Checking for an active step failed")
}
},
}
addCheckStepActiveFlags(checkStepActiveCmd)
return checkStepActiveCmd
}
func checkIfStepActive() error {
var pConfig config.Config
// load project config and defaults
projectConfig, err := initializeConfig(&pConfig)
if err != nil {
log.Entry().Errorf("Failed to load project config: %v", err)
}
stageConfigFile, err := checkStepActiveOptions.openFile(checkStepActiveOptions.stageConfigFile, GeneralConfig.GitHubAccessTokens)
if err != nil {
return errors.Wrapf(err, "config: open stage configuration file '%v' failed", checkStepActiveOptions.stageConfigFile)
}
defer stageConfigFile.Close()
// load and evaluate step conditions
stageConditions := &config.RunConfig{StageConfigFile: stageConfigFile}
err = stageConditions.InitRunConfig(projectConfig, nil, nil, nil, nil, doublestar.Glob, checkStepActiveOptions.openFile)
if err != nil {
return err
}
log.Entry().Debugf("RunSteps: %v", stageConditions.RunSteps)
if !stageConditions.RunSteps[checkStepActiveOptions.stageName][checkStepActiveOptions.stepName] {
return errors.Errorf("Step %s in stage %s is not active", checkStepActiveOptions.stepName, checkStepActiveOptions.stageName)
}
log.Entry().Infof("Step %s in stage %s is active", checkStepActiveOptions.stepName, checkStepActiveOptions.stageName)
return nil
}
func addCheckStepActiveFlags(cmd *cobra.Command) {
cmd.Flags().StringVar(&checkStepActiveOptions.stageConfigFile, "stageConfig", ".resources/piper-stage-config.yml",
"Default config of piper pipeline stages")
cmd.Flags().StringVar(&checkStepActiveOptions.stepName, "step", "", "Name of the step being checked")
cmd.Flags().StringVar(&checkStepActiveOptions.stageName, "stage", "", "Name of the stage in which the step being checked is")
cmd.MarkFlagRequired("step")
cmd.MarkFlagRequired("stage")
}
func initializeConfig(pConfig *config.Config) (*config.Config, error) {
projectConfigFile := getProjectConfigFile(GeneralConfig.CustomConfig)
customConfig, err := checkStepActiveOptions.openFile(projectConfigFile, GeneralConfig.GitHubAccessTokens)
if err != nil {
return nil, errors.Wrapf(err, "config: open configuration file '%v' failed", projectConfigFile)
}
defer customConfig.Close()
defaultConfig := []io.ReadCloser{}
for _, f := range GeneralConfig.DefaultConfig {
fc, err := checkStepActiveOptions.openFile(f, GeneralConfig.GitHubAccessTokens)
// only create error for non-default values
if err != nil && f != ".pipeline/defaults.yaml" {
return nil, errors.Wrapf(err, "config: getting defaults failed: '%v'", f)
}
if err == nil {
defaultConfig = append(defaultConfig, fc)
}
}
var flags map[string]interface{}
stepAliase := []config.Alias{}
filter := config.StepFilters{
All: []string{},
General: []string{},
Stages: []string{},
Steps: []string{},
Env: []string{},
}
_, err = pConfig.GetStepConfig(flags, "", customConfig, defaultConfig, GeneralConfig.IgnoreCustomDefaults, filter, nil, nil, nil, "", "",
stepAliase)
if err != nil {
return nil, errors.Wrap(err, "getting step config failed")
}
return pConfig, nil
}

View File

@ -0,0 +1,76 @@
package cmd
import (
"io"
"io/ioutil"
"strings"
"testing"
"github.com/spf13/cobra"
flag "github.com/spf13/pflag"
"github.com/stretchr/testify/assert"
)
func checkStepActiveOpenFileMock(name string, tokens map[string]string) (io.ReadCloser, error) {
var fileContent string
switch name {
case ".pipeline/defaults.yaml":
fileContent = `
general:
stages:
steps:`
case "stage-config.yml":
fileContent = `
stages:
testStage:
stepConditions:
testStep:
config: testConfig`
case ".pipeline/config.yml":
fileContent = `
steps:
testStep:
testConfig: 'testValue'`
default:
fileContent = ""
}
return ioutil.NopCloser(strings.NewReader(fileContent)), nil
}
func TestCheckStepActiveCommand(t *testing.T) {
cmd := CheckStepActiveCommand()
gotReq := []string{}
gotOpt := []string{}
cmd.Flags().VisitAll(func(pflag *flag.Flag) {
annotations, found := pflag.Annotations[cobra.BashCompOneRequiredFlag]
if found && annotations[0] == "true" {
gotReq = append(gotReq, pflag.Name)
} else {
gotOpt = append(gotOpt, pflag.Name)
}
})
t.Run("Required flags", func(t *testing.T) {
exp := []string{"stage", "step"}
assert.Equal(t, exp, gotReq, "required flags incorrect")
})
t.Run("Optional flags", func(t *testing.T) {
exp := []string{"stageConfig"}
assert.Equal(t, exp, gotOpt, "optional flags incorrect")
})
t.Run("Run", func(t *testing.T) {
t.Run("Success case", func(t *testing.T) {
checkStepActiveOptions.openFile = checkStepActiveOpenFileMock
checkStepActiveOptions.stageName = "testStage"
checkStepActiveOptions.stepName = "testStep"
checkStepActiveOptions.stageConfigFile = "stage-config.yml"
GeneralConfig.CustomConfig = ".pipeline/config.yml"
GeneralConfig.DefaultConfig = []string{".pipeline/defaults.yaml"}
cmd.Run(cmd, []string{})
})
})
}

View File

@ -159,6 +159,7 @@ func Execute() {
rootCmd.AddCommand(WritePipelineEnv())
rootCmd.AddCommand(ReadPipelineEnv())
rootCmd.AddCommand(InfluxWriteDataCommand())
rootCmd.AddCommand(CheckStepActiveCommand())
addRootFlags(rootCmd)

231
pkg/config/evaluation.go Normal file
View File

@ -0,0 +1,231 @@
package config
import (
"encoding/json"
"io"
"path"
"strings"
"github.com/pkg/errors"
)
const (
configCondition = "config"
configKeysCondition = "configKeys"
filePatternFromConfigCondition = "filePatternFromConfig"
filePatternCondition = "filePattern"
npmScriptsCondition = "npmScripts"
)
// EvaluateConditions validates stage conditions and updates runSteps in runConfig
func (r *RunConfig) evaluateConditions(config *Config, filters map[string]StepFilters, parameters map[string][]StepParameters,
secrets map[string][]StepSecrets, stepAliases map[string][]Alias, glob func(pattern string) (matches []string, err error)) error {
for stageName, stepConditions := range r.StageConfig.Stages {
runStep := map[string]bool{}
for stepName, stepCondition := range stepConditions.Conditions {
stepActive := false
stepConfig, err := r.getStepConfig(config, stageName, stepName, filters, parameters, secrets, stepAliases)
if err != nil {
return err
}
for conditionName, condition := range stepCondition {
var err error
switch conditionName {
case configCondition:
if stepActive, err = checkConfig(condition, stepConfig, stepName); err != nil {
return errors.Wrapf(err, "error: check config condition failed")
}
case configKeysCondition:
if stepActive, err = checkConfigKeys(condition, stepConfig, stepName); err != nil {
return errors.Wrapf(err, "error: check configKeys condition failed")
}
case filePatternFromConfigCondition:
if stepActive, err = checkForFilesWithPatternFromConfig(condition, stepConfig, stepName, glob); err != nil {
return errors.Wrapf(err, "error: check filePatternFromConfig condition failed")
}
case filePatternCondition:
if stepActive, err = checkForFilesWithPattern(condition, stepConfig, stepName, glob); err != nil {
return errors.Wrapf(err, "error: check filePattern condition failed")
}
case npmScriptsCondition:
if stepActive, err = checkForNpmScriptsInPackages(condition, stepConfig, stepName, glob, r.OpenFile); err != nil {
return errors.Wrapf(err, "error: check npmScripts condition failed")
}
default:
return errors.Errorf("unknown condition %s", conditionName)
}
if stepActive {
break
}
}
runStep[stepName] = stepActive
r.RunSteps[stageName] = runStep
}
}
return nil
}
func checkConfig(condition interface{}, config StepConfig, stepName string) (bool, error) {
switch condition := condition.(type) {
case string:
if configValue := stepConfigLookup(config.Config, stepName, condition); configValue != nil {
return true, nil
}
case map[string]interface{}:
for conditionConfigKey, conditionConfigValue := range condition {
configValue := stepConfigLookup(config.Config, stepName, conditionConfigKey)
if configValue == nil {
return false, nil
}
configValueStr, ok := configValue.(string)
if !ok {
return false, errors.Errorf("error: config value of %v to compare with is not a string", configValue)
}
condConfigValueArr, ok := conditionConfigValue.([]interface{})
if !ok {
return false, errors.Errorf("error: type assertion to []interface{} failed: %T", conditionConfigValue)
}
for _, item := range condConfigValueArr {
itemStr, ok := item.(string)
if !ok {
return false, errors.Errorf("error: type assertion to string failed: %T", conditionConfigValue)
}
if configValueStr == itemStr {
return true, nil
}
}
}
default:
return false, errors.Errorf("error: condidiion type invalid: %T, possible types: string, map[string]interface{}", condition)
}
return false, nil
}
func checkConfigKeys(condition interface{}, config StepConfig, stepName string) (bool, error) {
arrCondition, ok := condition.([]interface{})
if !ok {
return false, errors.Errorf("error: type assertion to []interface{} failed: %T", condition)
}
for _, configKey := range arrCondition {
if configValue := stepConfigLookup(config.Config, stepName, configKey.(string)); configValue != nil {
return true, nil
}
}
return false, nil
}
func checkForFilesWithPatternFromConfig(condition interface{}, config StepConfig, stepName string,
glob func(pattern string) (matches []string, err error)) (bool, error) {
filePatternConfig, ok := condition.(string)
if !ok {
return false, errors.Errorf("error: type assertion to string failed: %T", condition)
}
filePatternFromConfig := stepConfigLookup(config.Config, stepName, filePatternConfig)
if filePatternFromConfig == nil {
return false, nil
}
filePattern, ok := filePatternFromConfig.(string)
if !ok {
return false, errors.Errorf("error: type assertion to string failed: %T", filePatternFromConfig)
}
matches, err := glob(filePattern)
if err != nil {
return false, errors.Wrap(err, "error: failed to check if file-exists")
}
if len(matches) > 0 {
return true, nil
}
return false, nil
}
func checkForFilesWithPattern(condition interface{}, config StepConfig, stepName string,
glob func(pattern string) (matches []string, err error)) (bool, error) {
switch condition := condition.(type) {
case string:
filePattern := condition
matches, err := glob(filePattern)
if err != nil {
return false, errors.Wrap(err, "error: failed to check if file-exists")
}
if len(matches) > 0 {
return true, nil
}
case []interface{}:
filePatterns := condition
for _, filePattern := range filePatterns {
filePatternStr, ok := filePattern.(string)
if !ok {
return false, errors.Errorf("error: type assertion to string failed: %T", filePatternStr)
}
matches, err := glob(filePatternStr)
if err != nil {
return false, errors.Wrap(err, "error: failed to check if file-exists")
}
if len(matches) > 0 {
return true, nil
}
}
default:
return false, errors.Errorf("error: condidiion type invalid: %T, possible types: string, []interface{}", condition)
}
return false, nil
}
func checkForNpmScriptsInPackages(condition interface{}, config StepConfig, stepName string,
glob func(pattern string) (matches []string, err error), openFile func(s string, t map[string]string) (io.ReadCloser, error)) (bool, error) {
packages, err := glob("**/package.json")
if err != nil {
return false, errors.Wrap(err, "error: failed to check if file-exists")
}
for _, pack := range packages {
packDirs := strings.Split(path.Dir(pack), "/")
isNodeModules := false
for _, dir := range packDirs {
if dir == "node_modules" {
isNodeModules = true
break
}
}
if isNodeModules {
continue
}
jsonFile, err := openFile(pack, nil)
if err != nil {
return false, errors.Errorf("error: failed to open file %s: %v", pack, err)
}
defer jsonFile.Close()
packageJSON := map[string]interface{}{}
if err := json.NewDecoder(jsonFile).Decode(&packageJSON); err != nil {
return false, errors.Errorf("error: failed to unmarshal json file %s: %v", pack, err)
}
npmScripts, ok := packageJSON["scripts"]
if !ok {
continue
}
scriptsMap, ok := npmScripts.(map[string]interface{})
if !ok {
return false, errors.Errorf("error: type assertion to map[string]interface{} failed: %T", npmScripts)
}
switch condition := condition.(type) {
case string:
if _, ok := scriptsMap[condition]; ok {
return true, nil
}
case []interface{}:
for _, conditionNpmScript := range condition {
conditionNpmScriptStr, ok := conditionNpmScript.(string)
if !ok {
return false, errors.Errorf("error: type assertion to string failed: %T", conditionNpmScript)
}
if _, ok := scriptsMap[conditionNpmScriptStr]; ok {
return true, nil
}
}
default:
return false, errors.Errorf("error: condidiion type invalid: %T, possible types: string, []interface{}", condition)
}
}
return false, nil
}

View File

@ -0,0 +1,523 @@
package config
import (
"errors"
"io"
"io/ioutil"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func evaluateConditionsGlobMock(pattern string) ([]string, error) {
matches := []string{}
switch pattern {
case "**/conf.js":
matches = append(matches, "conf.js")
case "**/package.json":
matches = append(matches, "package.json", "package/node_modules/lib/package.json", "node_modules/package.json", "test/package.json")
}
return matches, nil
}
func evaluateConditionsOpenFileMock(name string, _ map[string]string) (io.ReadCloser, error) {
var fileContent io.ReadCloser
switch name {
case "package.json":
fileContent = ioutil.NopCloser(strings.NewReader(`
{
"scripts": {
"npmScript": "echo test",
"npmScript2": "echo test"
}
}
`))
case "_package.json":
fileContent = ioutil.NopCloser(strings.NewReader("wrong json format"))
case "test/package.json":
fileContent = ioutil.NopCloser(strings.NewReader("{}"))
}
return fileContent, nil
}
func Test_evaluateConditions(t *testing.T) {
tests := []struct {
name string
customConfig *Config
stageConfig io.ReadCloser
runStepsExpected map[string]map[string]bool
globFunc func(pattern string) ([]string, error)
wantErr bool
}{
{
name: "test config condition - success",
customConfig: &Config{
General: map[string]interface{}{
"testGeneral": "myVal1",
},
Stages: map[string]map[string]interface{}{
"testStage2": {
"testStage": "myVal2",
},
},
Steps: map[string]map[string]interface{}{
"thirdStep": {
"testStep1": "myVal3",
},
},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
config: testGeneral
testStage2:
stepConditions:
secondStep:
config: testStage
testStage3:
stepConditions:
thirdStep:
config: testStep1
forthStep:
config: testStep2
`)),
runStepsExpected: map[string]map[string]bool{
"testStage1": {"firstStep": true},
"testStage2": {"secondStep": true},
"testStage3": {
"thirdStep": true,
"forthStep": false,
},
},
wantErr: false,
},
{
name: "test config condition - wrong usage with list",
customConfig: &Config{
General: map[string]interface{}{},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
config:
- testGeneral
`)),
runStepsExpected: map[string]map[string]bool{},
wantErr: true,
},
{
name: "test config value condition - success",
customConfig: &Config{
General: map[string]interface{}{
"testGeneral": "myVal1",
},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{
"thirdStep": {
"testStep": "myVal3",
},
},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
config:
testGeneral:
- myValx
- myVal1
testStage2:
stepConditions:
secondStep:
config:
testStage:
- maValXyz
testStage3:
stepConditions:
thirdStep:
config:
testStep:
- myVal3
`)),
runStepsExpected: map[string]map[string]bool{
"testStage1": {"firstStep": true},
"testStage2": {"secondStep": false},
"testStage3": {"thirdStep": true},
},
wantErr: false,
},
{
name: "test configKey condition - success",
customConfig: &Config{
General: map[string]interface{}{
"myKey1_1": "myVal1_1",
},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{
"thirdStep": {
"myKey3_1": "myVal3_1",
},
},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
configKeys:
- myKey1_1
- myKey1_2
testStage2:
stepConditions:
secondStep:
configKeys:
- myKey2_1
testStage3:
stepConditions:
thirdStep:
configKeys:
- myKey3_1
`)),
runStepsExpected: map[string]map[string]bool{
"testStage1": {"firstStep": true},
"testStage2": {"secondStep": false},
"testStage3": {"thirdStep": true},
},
wantErr: false,
},
{
name: "test configKey condition - not list",
customConfig: &Config{
General: map[string]interface{}{
"myKey1_1": "myVal1_1",
},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
configKeys: myKey1_1
`)),
wantErr: true,
},
{
name: "test configKey condition - success",
customConfig: &Config{
General: map[string]interface{}{
"myKey1_1": "myVal1_1",
},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{
"thirdStep": {
"myKey3_1": "myVal3_1",
},
},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
configKeys:
- myKey1_1
- myKey1_2
testStage2:
stepConditions:
secondStep:
configKeys:
- myKey2_1
testStage3:
stepConditions:
thirdStep:
configKeys:
- myKey3_1
`)),
runStepsExpected: map[string]map[string]bool{
"testStage1": {"firstStep": true},
"testStage2": {"secondStep": false},
"testStage3": {"thirdStep": true},
},
wantErr: false,
},
{
name: "test filePattern condition - success",
globFunc: evaluateConditionsGlobMock,
customConfig: &Config{
General: map[string]interface{}{},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
filePattern: '**/conf.js'
secondStep:
filePattern: '**/conf.jsx'
`)),
runStepsExpected: map[string]map[string]bool{
"testStage1": {
"firstStep": true,
"secondStep": false,
},
},
wantErr: false,
},
{
name: "test filePattern condition - error while searching files by pattern",
customConfig: &Config{
General: map[string]interface{}{},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
filePattern: '**/conf.js'
`)),
runStepsExpected: map[string]map[string]bool{},
globFunc: func(pattern string) ([]string, error) {
return nil, errors.New("failed to check if file exists")
},
wantErr: true,
},
{
name: "test filePattern condition with list - success",
globFunc: evaluateConditionsGlobMock,
customConfig: &Config{
General: map[string]interface{}{},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
filePattern:
- '**/conf.js'
- 'myCollection.json'
secondStep:
filePattern:
- '**/conf.jsx'
`)),
runStepsExpected: map[string]map[string]bool{
"testStage1": {
"firstStep": true,
"secondStep": false,
},
},
wantErr: false,
},
{
name: "test filePattern condition with list - error while searching files by pattern",
customConfig: &Config{
General: map[string]interface{}{},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
filePattern:
- '**/conf.js'
- 'myCollection.json'
`)),
runStepsExpected: map[string]map[string]bool{},
globFunc: func(pattern string) ([]string, error) {
return nil, errors.New("failed to check if file exists")
},
wantErr: true,
},
{
name: "test filePatternFromConfig condition - success",
globFunc: evaluateConditionsGlobMock,
customConfig: &Config{
General: map[string]interface{}{},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{
"firstStep": {
"myVal1": "**/conf.js",
},
"thirdStep": {
"myVal3": "**/conf.jsx",
},
},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
filePatternFromConfig: myVal1
secondStep:
filePatternFromConfig: myVal2
thirdStep:
filePatternFromConfig: myVal3
`)),
runStepsExpected: map[string]map[string]bool{
"testStage1": {
"firstStep": true,
"secondStep": false,
"thirdStep": false,
},
},
wantErr: false,
},
{
name: "test filePatternFromConfig condition - error while searching files by pattern",
customConfig: &Config{
General: map[string]interface{}{},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{
"firstStep": {
"myVal1": "**/conf.js",
},
},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
filePatternFromConfig: myVal1
`)),
runStepsExpected: map[string]map[string]bool{},
globFunc: func(pattern string) ([]string, error) {
return nil, errors.New("failed to check if file exists")
},
wantErr: true,
},
{
name: "test npmScripts condition - success",
globFunc: evaluateConditionsGlobMock,
customConfig: &Config{
General: map[string]interface{}{},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
npmScripts: 'npmScript'
secondStep:
npmScripts: 'npmScript1'
`)),
runStepsExpected: map[string]map[string]bool{
"testStage1": {
"firstStep": true,
"secondStep": false,
},
},
wantErr: false,
},
{
name: "test npmScripts condition with list - success",
globFunc: evaluateConditionsGlobMock,
customConfig: &Config{
General: map[string]interface{}{},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
npmScripts:
- 'npmScript'
- 'npmScript2'
secondStep:
npmScripts:
- 'npmScript3'
- 'npmScript4'
`)),
runStepsExpected: map[string]map[string]bool{
"testStage1": {
"firstStep": true,
"secondStep": false,
},
},
wantErr: false,
},
{
name: "test npmScripts condition - json with wrong format",
customConfig: &Config{
General: map[string]interface{}{},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
npmScripts:
- 'npmScript3'
`)),
runStepsExpected: map[string]map[string]bool{},
globFunc: func(pattern string) ([]string, error) {
return []string{"_package.json"}, nil
},
wantErr: true,
},
{
name: "test npmScripts condition - error while searching package.json",
customConfig: &Config{
General: map[string]interface{}{},
Stages: map[string]map[string]interface{}{},
Steps: map[string]map[string]interface{}{},
},
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
npmScripts:
- 'npmScript3'
`)),
runStepsExpected: map[string]map[string]bool{},
globFunc: func(pattern string) ([]string, error) {
return nil, errors.New("failed to check if file exists")
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
runConfig := RunConfig{
StageConfigFile: tt.stageConfig,
RunSteps: map[string]map[string]bool{},
OpenFile: evaluateConditionsOpenFileMock,
}
err := runConfig.loadConditions()
assert.NoError(t, err)
err = runConfig.evaluateConditions(tt.customConfig, nil, nil, nil, nil, tt.globFunc)
if tt.wantErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, tt.runStepsExpected, runConfig.RunSteps)
}
})
}
}

119
pkg/config/run.go Normal file
View File

@ -0,0 +1,119 @@
package config
import (
"io"
"io/ioutil"
"github.com/ghodss/yaml"
"github.com/pkg/errors"
)
// RunConfig ...
type RunConfig struct {
StageConfigFile io.ReadCloser
StageConfig StageConfig
RunSteps map[string]map[string]bool
OpenFile func(s string, t map[string]string) (io.ReadCloser, error)
}
type StageConfig struct {
Stages map[string]StepConditions `json:"stages,omitempty"`
}
type StepConditions struct {
Conditions map[string]map[string]interface{} `json:"stepConditions,omitempty"`
}
// InitRunConfig ...
func (r *RunConfig) InitRunConfig(config *Config, filters map[string]StepFilters, parameters map[string][]StepParameters,
secrets map[string][]StepSecrets, stepAliases map[string][]Alias, glob func(pattern string) (matches []string, err error),
openFile func(s string, t map[string]string) (io.ReadCloser, error)) error {
r.OpenFile = openFile
r.RunSteps = map[string]map[string]bool{}
if len(r.StageConfig.Stages) == 0 {
if err := r.loadConditions(); err != nil {
return errors.Wrap(err, "failed to load pipeline run conditions")
}
}
err := r.evaluateConditions(config, filters, parameters, secrets, stepAliases, glob)
if err != nil {
return errors.Wrap(err, "failed to evaluate step conditions: %v")
}
return nil
}
// ToDo: optimize parameter handling
func (r *RunConfig) getStepConfig(config *Config, stageName, stepName string, filters map[string]StepFilters,
parameters map[string][]StepParameters, secrets map[string][]StepSecrets, stepAliases map[string][]Alias) (StepConfig, error) {
// no support for flag values and envParameters
// so far not considered necessary
flagValues := map[string]interface{}{} // args of step from pipeline_generated.yml
envParameters := map[string]interface{}{}
// parameters via paramJSON not supported
// not considered releavant for pipeline yaml syntax resolution
paramJSON := ""
return config.GetStepConfig(flagValues, paramJSON, nil, nil, false, filters[stepName], parameters[stepName], secrets[stepName],
envParameters, stageName, stepName, stepAliases[stepName])
}
func (r *RunConfig) loadConditions() error {
defer r.StageConfigFile.Close()
content, err := ioutil.ReadAll(r.StageConfigFile)
if err != nil {
return errors.Wrapf(err, "error: failed to read the stageConfig file")
}
err = yaml.Unmarshal(content, &r.StageConfig)
if err != nil {
return errors.Errorf("format of configuration is invalid %q: %v", content, err)
}
return nil
}
func stepConfigLookup(m map[string]interface{}, stepName, key string) interface{} {
// flat map: key is on top level
if m[key] != nil {
return m[key]
}
// lookup for step config with following format
// general:
// <key>: <value>
// stages:
// <stepName>:
// <key>: <value>
// steps:
// <stepName>:
// <key>: <value>
if m["general"] != nil {
general := m["general"].(map[string]interface{})
if general[key] != nil {
return general[key]
}
}
if m["stages"] != nil {
stages := m["stages"].(map[string]interface{})
if stages[stepName] != nil {
stageStepConfig := stages[stepName].(map[string]interface{})
if stageStepConfig[key] != nil {
return stageStepConfig[key]
}
}
}
if m["steps"] != nil {
steps := m["steps"].(map[string]interface{})
if steps[stepName] != nil {
stepConfig := steps[stepName].(map[string]interface{})
if stepConfig[key] != nil {
return stepConfig[key]
}
}
}
return nil
}

266
pkg/config/run_test.go Normal file
View File

@ -0,0 +1,266 @@
package config
import (
"io"
"io/ioutil"
"reflect"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func initRunConfigGlobMock(pattern string) ([]string, error) {
matches := []string{}
switch pattern {
case "**/file1":
matches = append(matches, "file1")
case "directory/file2":
matches = append(matches, "file2")
}
return matches, nil
}
func TestInitRunConfig(t *testing.T) {
tests := []struct {
name string
customConfig io.ReadCloser
stageConfig io.ReadCloser
runStepsExpected map[string]map[string]bool
wantErr bool
}{
{
name: "init run config with config condition - success",
customConfig: ioutil.NopCloser(strings.NewReader(`
general:
testGeneral: 'myVal1'
stages:
testStage2:
testStage: 'myVal2'
steps:
thirdStep:
testStep: 'myVal3'
`)),
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
config: testGeneral
testStage2:
stepConditions:
secondStep:
config: testStage
testStage3:
stepConditions:
thirdStep:
config: testStep
`)),
runStepsExpected: map[string]map[string]bool{
"testStage1": {
"firstStep": true,
},
"testStage2": {
"secondStep": true,
},
"testStage3": {
"thirdStep": true,
},
},
wantErr: false,
},
{
name: "init run config with filePattern condition - success",
customConfig: ioutil.NopCloser(strings.NewReader(`
general:
testGeneral: 'myVal1'
stages:
testStage2:
testStage: 'myVal2'
steps:
thirdStep:
testStep: 'myVal3'
`)),
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage1:
stepConditions:
firstStep:
filePattern: "**/file1"
testStage2:
stepConditions:
secondStep:
filePattern: "directory/file2"
testStage3:
stepConditions:
thirdStep:
filePattern: "file3"
`)),
runStepsExpected: map[string]map[string]bool{
"testStage1": {
"firstStep": true,
},
"testStage2": {
"secondStep": true,
},
"testStage3": {
"thirdStep": false,
},
},
wantErr: false,
},
{
name: "init run config - unknown condition in stage config",
customConfig: ioutil.NopCloser(strings.NewReader(`
steps:
testStep:
testConfig: 'testVal'
`)),
stageConfig: ioutil.NopCloser(strings.NewReader(`
stages:
testStage:
stepConditions:
testStep:
wrongCondition: "condVal"
`)),
runStepsExpected: map[string]map[string]bool{},
wantErr: true,
},
{
name: "init run config - load conditions with invalid format",
stageConfig: ioutil.NopCloser(strings.NewReader("wrong stage config format")),
runStepsExpected: map[string]map[string]bool{},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
runConfig := RunConfig{StageConfigFile: tt.stageConfig}
filter := StepFilters{All: []string{}, General: []string{}, Stages: []string{}, Steps: []string{}, Env: []string{}}
projectConfig := Config{}
_, err := projectConfig.GetStepConfig(map[string]interface{}{}, "", tt.customConfig,
[]io.ReadCloser{}, false, filter, nil, nil, nil, "", "", []Alias{})
assert.NoError(t, err)
err = runConfig.InitRunConfig(&projectConfig, nil, nil, nil, nil, initRunConfigGlobMock, nil)
if tt.wantErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, tt.runStepsExpected, runConfig.RunSteps)
}
})
}
}
func TestRunConfigLoadConditions(t *testing.T) {
stageConfigContent := `stages:
'testStage1':
stepConditions:
firstStep:
filePattern: '**/my.file'
`
t.Run("load conditions - file of invalid format", func(t *testing.T) {
runConfig := &RunConfig{StageConfigFile: ioutil.NopCloser(strings.NewReader("-- {{ \\ wrong } file format }"))}
err := runConfig.loadConditions()
assert.Error(t, err, "format of configuration is invalid")
})
t.Run("load conditions - success", func(t *testing.T) {
runConfig := &RunConfig{StageConfigFile: ioutil.NopCloser(strings.NewReader(stageConfigContent))}
err := runConfig.loadConditions()
assert.NoError(t, err)
condition := map[string]interface{}{
"filePattern": "**/my.file",
}
assert.Equal(t, 1, len(runConfig.StageConfig.Stages))
assert.Equal(t, 1, len(runConfig.StageConfig.Stages["testStage1"].Conditions))
assert.Equal(t, condition, runConfig.StageConfig.Stages["testStage1"].Conditions["firstStep"])
})
}
func Test_stepConfigLookup(t *testing.T) {
testConfig := map[string]interface{}{
"general": map[string]interface{}{
"generalKey": "generalValue",
},
"stages": map[string]interface{}{
"testStep": map[string]interface{}{
"stagesKey": "stagesValue",
},
},
"steps": map[string]interface{}{
"testStep": map[string]interface{}{
"stepKey": "stepValue",
"stepKeyStringSlice": []string{"val1", "val2"},
},
},
"configKey": "configValue",
}
type args struct {
m map[string]interface{}
stepName string
key string
}
tests := []struct {
name string
args args
want interface{}
}{
{
name: "empty map",
args: args{nil, "", ""},
want: nil,
},
{
name: "key not in map, invalid stepName",
args: args{testConfig, "some step", "some key"},
want: nil,
},
{
name: "key not in map, valid stepName",
args: args{testConfig, "testStep", "some key"},
want: nil,
},
{
name: "key in map under general",
args: args{testConfig, "some step", "generalKey"},
want: "generalValue",
},
{
name: "key in map under stages",
args: args{testConfig, "testStep", "stagesKey"},
want: "stagesValue",
},
{
name: "key in map under general",
args: args{testConfig, "testStep", "stepKey"},
want: "stepValue",
},
{
name: "key in map under general",
args: args{testConfig, "testStep", "stepKeyStringSlice"},
want: []string{"val1", "val2"},
},
{
name: "key in map on top level string",
args: args{testConfig, "", "configKey"},
want: "configValue",
},
{
name: "key in map on top level map",
args: args{testConfig, "", "general"},
want: map[string]interface{}{"generalKey": "generalValue"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := stepConfigLookup(tt.args.m, tt.args.stepName, tt.args.key); !reflect.DeepEqual(got, tt.want) {
t.Errorf("stepConfigLookup() = %v, want %v", got, tt.want)
}
})
}
}