Extend conditions for step and stage activation (#1955)

Co-authored-by: Stephan Aßmus <stephan.assmus@sap.com>
2025-02-09 13:47:31 +02:00 · 2020-08-27 15:49:09 +02:00 · 2020-08-27 15:49:09 +02:00 · 139e34cd37
commit 139e34cd37
parent c6e409dfd9
5 changed files with 419 additions and 94 deletions
--- a/resources/com.sap.piper/pipeline/cloudSdkStageDefaults.yml
+++ b/resources/com.sap.piper/pipeline/cloudSdkStageDefaults.yml
@ -48,7 +48,10 @@ stages:
  backendIntegrationTests:
    stepConditions:
      npmExecuteScripts:
-        configKeys:
-          - 'runScripts'
+        npmScripts: 'ci-it-backend'
      mavenExecuteIntegration:
        filePattern: 'integration-tests/pom.xml'
+  frontendIntegrationTests:
+    stepConditions:
+      npmExecuteScripts:
+        npmScripts: 'ci-it-frontend'
--- a/test/groovy/PiperStageWrapperTest.groovy
+++ b/test/groovy/PiperStageWrapperTest.groovy
@ -264,7 +264,7 @@ class PiperStageWrapperTest extends BasePiperTest {
        })

        nullScript.commonPipelineEnvironment.gitBranch = 'testBranch'
-        binding.setVariable('env', [PIPER_DISABLE_EXTENSIONS: 'true'])
+        nullScript.env = [PIPER_DISABLE_EXTENSIONS: 'true']
        stepRule.step.piperStageWrapper(
            script: nullScript,
            juStabUtils: utils,
--- a/test/groovy/templates/PiperInitRunStageConfigurationTest.groovy
+++ b/test/groovy/templates/PiperInitRunStageConfigurationTest.groovy
@ -76,15 +76,9 @@ steps: {}
            stageConfigResource: 'testDefault.yml'
        )

-        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.keySet(),
-            allOf(
-                containsInAnyOrder(
-                    'testStage2',
-                    'testStage3'
-                ),
-                hasSize(2)
-            )
-        )
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(false))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage2, is(true))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage3, is(true))
    }


@ -128,16 +122,9 @@ steps: {}
            stageConfigResource: 'testDefault.yml'
        )

-        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.keySet(),
-            allOf(
-                containsInAnyOrder(
-                    'testStage1',
-                    'testStage2',
-                    'testStage3'
-                ),
-                hasSize(3)
-            )
-        )
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(true))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage2, is(true))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage3, is(true))

        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true))
        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage2.secondStep, is(true))
@ -192,15 +179,9 @@ steps: {}
            stageConfigResource: 'testDefault.yml'
        )

-        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.keySet(),
-            allOf(
-                containsInAnyOrder(
-                    'testStage1',
-                    'testStage3'
-                ),
-                hasSize(2)
-            )
-        )
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(true))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage2, is(false))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage3, is(true))

        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true))
        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage2?.secondStep, is(false))
@ -251,15 +232,9 @@ steps: {}
            stageConfigResource: 'testDefault.yml'
        )

-        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.keySet(),
-            allOf(
-                containsInAnyOrder(
-                    'testStage1',
-                    'testStage3'
-                ),
-                hasSize(2)
-            )
-        )
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(true))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage2, is(false))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage3, is(true))

        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true))
        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage2?.secondStep, is(false))
@ -308,6 +283,41 @@ steps: {}

    }

+    @Test
+    void testConditionFilePatternWithList() {
+        helper.registerAllowedMethod('libraryResource', [String.class], {s ->
+            if(s == 'testDefault.yml') {
+                return '''
+stages:
+  testStage1:
+    stepConditions:
+      firstStep:
+        filePattern:
+         - \'**/conf.js\'
+         - \'myCollection.json\'
+      secondStep:
+        filePattern: \'**/conf.jsx\'
+
+'''
+            } else {
+                return '''
+general: {}
+steps: {}
+'''
+            }
+        })
+
+        jsr.step.piperInitRunStageConfiguration(
+            script: nullScript,
+            juStabUtils: utils,
+            stageConfigResource: 'testDefault.yml'
+        )
+
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.secondStep, is(false))
+
+    }
+
    @Test
    void testConditionFilePatternFromConfig() {
        helper.registerAllowedMethod('libraryResource', [String.class], {s ->
@ -397,15 +407,8 @@ steps: {}
            stageConfigResource: 'com.sap.piper/pipeline/stageDefaults.yml'
        )

-        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.keySet(),
-            allOf(
-                containsInAnyOrder(
-                    'Acceptance',
-                    'Integration'
-                ),
-                hasSize(2)
-            )
-        )
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.Acceptance, is(true))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.Integration, is(true))

    }

@ -450,4 +453,229 @@ steps: {}
        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.Acceptance, is(true))

    }
+
+    @Test
+    void testConditionNpmScripts() {
+        helper.registerAllowedMethod('libraryResource', [String.class], {s ->
+            if(s == 'testDefault.yml') {
+                return '''
+stages:
+  testStage1:
+    stepConditions:
+      firstStep:
+        npmScripts: \'npmScript\'
+      secondStep:
+        filePattern: \'**/conf.jsx\'
+
+'''
+            } else {
+                return '''
+general: {}
+steps: {}
+'''
+            }
+        })
+
+        helper.registerAllowedMethod('findFiles', [Map], {m ->
+            if(m.glob == '**/package.json') {
+                return [new File("package.json")].toArray()
+            } else {
+                return []
+            }
+        })
+
+        helper.registerAllowedMethod('readJSON', [Map], { m ->
+            if (m.file == 'package.json') {
+                return [scripts: [npmScript: "echo test",
+                                  npmScript2: "echo test"]]
+            } else {
+                return [:]
+            }
+        })
+
+        jsr.step.piperInitRunStageConfiguration(
+            script: nullScript,
+            juStabUtils: utils,
+            stageConfigResource: 'testDefault.yml'
+        )
+
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.secondStep, is(false))
+
+    }
+
+    @Test
+    void testConditionNpmScriptsWithList() {
+        helper.registerAllowedMethod('libraryResource', [String.class], {s ->
+                    if(s == 'testDefault.yml') {
+                        return '''
+stages:
+  testStage1:
+    stepConditions:
+      firstStep:
+        npmScripts:
+         - \'npmScript\'
+         - \'npmScript2\'
+      secondStep:
+        filePattern: \'**/conf.jsx\'
+
+'''
+                    } else {
+                        return '''
+general: {}
+steps: {}
+'''
+            }
+        })
+
+        helper.registerAllowedMethod('findFiles', [Map], {m ->
+            if(m.glob == '**/package.json') {
+                return [new File("package.json")].toArray()
+            } else {
+                return []
+            }
+        })
+
+        helper.registerAllowedMethod('readJSON', [Map], { m ->
+            if (m.file == 'package.json') {
+                return [scripts: [npmScript: "echo test",
+                                  npmScript2: "echo test"]]
+            } else {
+                return [:]
+            }
+        })
+
+        jsr.step.piperInitRunStageConfiguration(
+            script: nullScript,
+            juStabUtils: utils,
+            stageConfigResource: 'testDefault.yml'
+        )
+
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.secondStep, is(false))
+
+    }
+
+    @Test
+    void testConditionOnlyProductiveBranchOnNonProductiveBranch() {
+        helper.registerAllowedMethod('libraryResource', [String.class], {s ->
+            if(s == 'testDefault.yml') {
+                return '''
+stages:
+  testStage1:
+    onlyProductiveBranch: true
+    stepConditions:
+      firstStep:
+        filePattern: \'**/conf.js\'
+'''
+            } else {
+                return '''
+general: {}
+steps: {}
+'''
+            }
+        })
+
+        binding.variables.env.BRANCH_NAME = 'test'
+
+        jsr.step.piperInitRunStageConfiguration(
+            script: nullScript,
+            juStabUtils: utils,
+            stageConfigResource: 'testDefault.yml',
+            productiveBranch: 'master'
+        )
+
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(false))
+    }
+
+    @Test
+    void testConditionOnlyProductiveBranchOnProductiveBranch() {
+        helper.registerAllowedMethod('libraryResource', [String.class], {s ->
+            if(s == 'testDefault.yml') {
+                return '''
+stages:
+  testStage1:
+    onlyProductiveBranch: true
+    stepConditions:
+      firstStep:
+        filePattern: \'**/conf.js\'
+'''
+            } else {
+                return '''
+general: {}
+steps: {}
+'''
+            }
+        })
+
+        binding.variables.env.BRANCH_NAME = 'test'
+
+        jsr.step.piperInitRunStageConfiguration(
+            script: nullScript,
+            juStabUtils: utils,
+            stageConfigResource: 'testDefault.yml',
+            productiveBranch: 'test'
+        )
+
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(true))
+    }
+
+    @Test
+    void testStageExtensionExists() {
+        helper.registerAllowedMethod('libraryResource', [String.class], {s ->
+            if(s == 'testDefault.yml') {
+                return '''
+stages:
+  testStage1:
+    extensionExists: true
+  testStage2:
+    extensionExists: true
+  testStage3:
+    extensionExists: false
+  testStage4:
+    extensionExists: 'false'
+  testStage5:
+    dummy: true
+'''
+            } else {
+                return '''
+general:
+  projectExtensionsDirectory: './extensions/'
+steps: {}
+'''
+            }
+        })
+
+        helper.registerAllowedMethod('fileExists', [String], {path ->
+            switch (path) {
+                case './extensions/testStage1.groovy':
+                    return true
+                case './extensions/testStage2.groovy':
+                    return false
+                case './extensions/testStage3.groovy':
+                    return true
+                case './extensions/testStage4.groovy':
+                    return true
+                case './extensions/testStage5.groovy':
+                    return true
+                default:
+                    return false
+            }
+        })
+
+        nullScript.piperStageWrapper = [:]
+        nullScript.piperStageWrapper.allowExtensions = {script -> return true}
+
+        jsr.step.piperInitRunStageConfiguration(
+            script: nullScript,
+            juStabUtils: utils,
+            stageConfigResource: 'testDefault.yml'
+        )
+
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(true))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage2, is(false))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage3, is(false))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage4, is(false))
+        assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage5, is(false))
+    }
 }
--- a/vars/piperInitRunStageConfiguration.groovy
+++ b/vars/piperInitRunStageConfiguration.groovy
@ -13,7 +13,20 @@ import groovy.transform.Field
     * Print more detailed information into the log.
     * @possibleValues `true`, `false`
     */
-    'verbose'
+    'verbose',
+    /**
+     * The branch used as productive branch, defaults to master.
+     */
+    'productiveBranch',
+    /**
+     * Location for individual stage extensions.
+     */
+    'projectExtensionsDirectory',
+    /**
+     * Location for global extensions.
+     */
+    'globalExtensionsDirectory'
+
 ]

@Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS.plus([
@ -50,60 +63,52 @@ void call(Map parameters = [:]) {
    //handling of stage and step activation
    config.stages.each {stage ->

-        //activate stage if stage configuration is available
-        if (ConfigurationLoader.stageConfiguration(script, stage.getKey())) {
-            script.commonPipelineEnvironment.configuration.runStage[stage.getKey()] = true
-        }
-        //-------------------------------------------------------------------------------
-        //detailed handling of step and stage activation based on conditions
-        script.commonPipelineEnvironment.configuration.runStep[stage.getKey()] = [:]
-        def currentStage = stage.getKey()
-        stage.getValue().stepConditions.each {step ->
-            def stepActive = false
+        String currentStage = stage.getKey()
+        script.commonPipelineEnvironment.configuration.runStep[currentStage] = [:]
+
+        // Always test step conditions in order to fill runStep[currentStage] map
+        boolean anyStepConditionTrue = false
+        stage.getValue().stepConditions?.each {step ->
+            boolean stepActive = false
+            String stepName = step.getKey()
            step.getValue().each {condition ->
-                Map stepConfig = script.commonPipelineEnvironment.getStepConfiguration(step.getKey(), currentStage)
+                Map stepConfig = script.commonPipelineEnvironment.getStepConfiguration(stepName, currentStage)
                switch(condition.getKey()) {
                    case 'config':
-                        if (condition.getValue() instanceof Map) {
-                            condition.getValue().each {configCondition ->
-                                if (MapUtils.getByPath(stepConfig, configCondition.getKey()) in configCondition.getValue()) {
-                                    stepActive = true
-                                }
-                            }
-                        } else if (MapUtils.getByPath(stepConfig, condition.getValue())) {
-                            stepActive = true
-                        }
+                        stepActive = stepActive || checkConfig(condition, stepConfig)
                        break
                    case 'configKeys':
-                        if (condition.getValue() instanceof List) {
-                            condition.getValue().each {configKey ->
-                                if (MapUtils.getByPath(stepConfig, configKey)) {
-                                    stepActive = true
-                                }
-                            }
-                        } else if (MapUtils.getByPath(stepConfig, condition.getValue())) {
-                            stepActive = true
-                        }
+                        stepActive = stepActive || checkConfigKeys(condition, stepConfig)
                        break
                    case 'filePatternFromConfig':
-                        def conditionValue = MapUtils.getByPath(stepConfig, condition.getValue())
-                        if (conditionValue && findFiles(glob: conditionValue)) {
-                            stepActive = true
-                        }
+                        stepActive = stepActive || checkForFilesWithPatternFromConfig(script, condition, stepConfig)
                        break
                    case 'filePattern':
-                        if (findFiles(glob: condition.getValue())) {
-                            stepActive = true
-                        }
+                        stepActive = stepActive || checkForFilesWithPattern(script, condition)
+                        break
+                    case 'npmScripts':
+                        stepActive = stepActive || checkForNpmScriptsInPackages(script, condition)
                        break
                }
            }
-            script.commonPipelineEnvironment.configuration.runStep."${stage.getKey()}"."${step.getKey()}" = stepActive
-
-            //make sure that also related stage is activated if steps are active
-            if (stepActive) script.commonPipelineEnvironment.configuration.runStage[stage.getKey()] = true
+            script.commonPipelineEnvironment.configuration.runStep[currentStage][stepName] = stepActive

+            anyStepConditionTrue = anyStepConditionTrue || stepActive
        }
+
+        boolean runStage
+        if (stage.getValue().onlyProductiveBranch && (config.productiveBranch != env.BRANCH_NAME)) {
+            runStage = false
+        } else if (ConfigurationLoader.stageConfiguration(script, currentStage)) {
+            //activate stage if stage configuration is available
+            runStage = true
+        } else if (stage.getValue().extensionExists == true) {
+            runStage = anyStepConditionTrue || checkExtensionExists(script, config, currentStage)
+        } else {
+            runStage = anyStepConditionTrue
+        }
+
+        script.commonPipelineEnvironment.configuration.runStage[currentStage] = runStage
    }

    if (config.verbose) {
@ -111,3 +116,92 @@ void call(Map parameters = [:]) {
        echo "[${STEP_NAME}] Debug - Run Step Configuration: ${script.commonPipelineEnvironment.configuration.runStep}"
    }
 }
+
+private static boolean checkExtensionExists(Script script, Map config, String stageName) {
+    if (!script.piperStageWrapper.allowExtensions(script)) {
+        return false
+    }
+    // NOTE: These keys exist in "config" if they are configured in the general section of the project
+    // config or the defaults. However, in piperStageWrapper, these keys could also be configured for
+    // the step "piperStageWrapper" to be effective. Don't know if this should be considered here for consistency.
+    if (!config.globalExtensionsDirectory && !config.projectExtensionsDirectory) {
+        return false
+    }
+    def projectInterceptorFile = "${config.projectExtensionsDirectory}${stageName}.groovy"
+    def globalInterceptorFile = "${config.globalExtensionsDirectory}${stageName}.groovy"
+    return script.fileExists(projectInterceptorFile) || script.fileExists(globalInterceptorFile)
+}
+
+private static boolean checkConfig(def condition, Map stepConfig) {
+    Boolean configExists = false
+    if (condition.getValue() instanceof Map) {
+        condition.getValue().each {configCondition ->
+            if (MapUtils.getByPath(stepConfig, configCondition.getKey()) in configCondition.getValue()) {
+                configExists = true
+            }
+        }
+    } else if (MapUtils.getByPath(stepConfig, condition.getValue())) {
+        configExists = true
+    }
+    return configExists
+}
+
+private static boolean checkConfigKeys(def condition, Map stepConfig) {
+    Boolean configKeyExists = false
+    if (condition.getValue() instanceof List) {
+        condition.getValue().each { configKey ->
+            if (MapUtils.getByPath(stepConfig, configKey)) {
+                configKeyExists = true
+            }
+        }
+    } else if (MapUtils.getByPath(stepConfig, condition.getValue())) {
+        configKeyExists = true
+    }
+    return configKeyExists
+}
+
+private static boolean checkForFilesWithPatternFromConfig (Script script, def condition, Map stepConfig) {
+    def conditionValue = MapUtils.getByPath(stepConfig, condition.getValue())
+    if (conditionValue && script.findFiles(glob: conditionValue)) {
+        return true
+    }
+    return false
+}
+
+private static boolean checkForFilesWithPattern (Script script, def condition) {
+    Boolean filesExist = false
+    if (condition.getValue() instanceof List) {
+        condition.getValue().each {configKey ->
+            if (script.findFiles(glob: configKey)) {
+                filesExist = true
+            }
+        }
+    } else {
+        if (script.findFiles(glob: condition.getValue())) {
+            filesExist = true
+        }
+    }
+    return filesExist
+}
+
+private static boolean checkForNpmScriptsInPackages (Script script, def condition) {
+    def packages = script.findFiles(glob: '**/package.json', excludes: '**/node_modules/**')
+    Boolean npmScriptExists = false
+    for (int i = 0; i < packages.size(); i++) {
+        String packageJsonPath = packages[i].path
+        Map packageJson = script.readJSON file: packageJsonPath
+        Map npmScripts = packageJson.scripts ?: [:]
+        if (condition.getValue() instanceof List) {
+            condition.getValue().each { configKey ->
+                if (npmScripts[configKey]) {
+                    npmScriptExists = true
+                }
+            }
+        } else {
+            if (npmScripts[condition.getValue()]) {
+                npmScriptExists = true
+            }
+        }
+    }
+    return npmScriptExists
+}
--- a/vars/piperStageWrapper.groovy
+++ b/vars/piperStageWrapper.groovy
@ -109,7 +109,7 @@ private void executeStage(script, originalStage, stageName, config, utils, telem
        def body = originalStage

        // First, check if a global extension exists via a dedicated repository
-        if (globalExtensions && allowExtensions()) {
+        if (globalExtensions && allowExtensions(script)) {
            echo "[${STEP_NAME}] Found global interceptor '${globalInterceptorFile}' for ${stageName}."
            // If we call the global interceptor, we will pass on originalStage as parameter
            DebugReport.instance.globalExtensions.put(stageName, "Overwrites")
@ -124,7 +124,7 @@ private void executeStage(script, originalStage, stageName, config, utils, telem
        }

        // Second, check if a project extension (within the same repository) exists
-        if (projectExtensions && allowExtensions()) {
+        if (projectExtensions && allowExtensions(script)) {
            echo "[${STEP_NAME}] Running project interceptor '${projectInterceptorFile}' for ${stageName}."
            // If we call the project interceptor, we will pass on body as parameter which contains either originalStage or the repository interceptor
            if (projectExtensions && globalExtensions) {
@ -225,6 +225,6 @@ private boolean isOldInterceptorInterfaceUsed(Script interceptor) {
    return method != null
 }

-private boolean allowExtensions() {
-    return env.PIPER_DISABLE_EXTENSIONS == null || Boolean.valueOf(env.PIPER_DISABLE_EXTENSIONS) == false
+private static boolean allowExtensions(Script script) {
+    return script.env.PIPER_DISABLE_EXTENSIONS == null || Boolean.valueOf(script.env.PIPER_DISABLE_EXTENSIONS) == false
 }