From 139e34cd3701a410d7b64ba747bac64b6cb47e08 Mon Sep 17 00:00:00 2001 From: Kevin Hudemann Date: Thu, 27 Aug 2020 15:49:09 +0200 Subject: [PATCH] Extend conditions for step and stage activation (#1955) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Stephan Aßmus --- .../pipeline/cloudSdkStageDefaults.yml | 7 +- test/groovy/PiperStageWrapperTest.groovy | 2 +- .../PiperInitRunStageConfigurationTest.groovy | 320 +++++++++++++++--- vars/piperInitRunStageConfiguration.groovy | 176 +++++++--- vars/piperStageWrapper.groovy | 8 +- 5 files changed, 419 insertions(+), 94 deletions(-) diff --git a/resources/com.sap.piper/pipeline/cloudSdkStageDefaults.yml b/resources/com.sap.piper/pipeline/cloudSdkStageDefaults.yml index 2c4896ddb..3b4120ff1 100644 --- a/resources/com.sap.piper/pipeline/cloudSdkStageDefaults.yml +++ b/resources/com.sap.piper/pipeline/cloudSdkStageDefaults.yml @@ -48,7 +48,10 @@ stages: backendIntegrationTests: stepConditions: npmExecuteScripts: - configKeys: - - 'runScripts' + npmScripts: 'ci-it-backend' mavenExecuteIntegration: filePattern: 'integration-tests/pom.xml' + frontendIntegrationTests: + stepConditions: + npmExecuteScripts: + npmScripts: 'ci-it-frontend' diff --git a/test/groovy/PiperStageWrapperTest.groovy b/test/groovy/PiperStageWrapperTest.groovy index 41173a77f..195b41425 100644 --- a/test/groovy/PiperStageWrapperTest.groovy +++ b/test/groovy/PiperStageWrapperTest.groovy @@ -264,7 +264,7 @@ class PiperStageWrapperTest extends BasePiperTest { }) nullScript.commonPipelineEnvironment.gitBranch = 'testBranch' - binding.setVariable('env', [PIPER_DISABLE_EXTENSIONS: 'true']) + nullScript.env = [PIPER_DISABLE_EXTENSIONS: 'true'] stepRule.step.piperStageWrapper( script: nullScript, juStabUtils: utils, diff --git a/test/groovy/templates/PiperInitRunStageConfigurationTest.groovy b/test/groovy/templates/PiperInitRunStageConfigurationTest.groovy index 9cda73fd3..acfcfa276 100644 --- a/test/groovy/templates/PiperInitRunStageConfigurationTest.groovy +++ b/test/groovy/templates/PiperInitRunStageConfigurationTest.groovy @@ -76,15 +76,9 @@ steps: {} stageConfigResource: 'testDefault.yml' ) - assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.keySet(), - allOf( - containsInAnyOrder( - 'testStage2', - 'testStage3' - ), - hasSize(2) - ) - ) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(false)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage2, is(true)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage3, is(true)) } @@ -128,16 +122,9 @@ steps: {} stageConfigResource: 'testDefault.yml' ) - assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.keySet(), - allOf( - containsInAnyOrder( - 'testStage1', - 'testStage2', - 'testStage3' - ), - hasSize(3) - ) - ) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(true)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage2, is(true)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage3, is(true)) assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true)) assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage2.secondStep, is(true)) @@ -192,15 +179,9 @@ steps: {} stageConfigResource: 'testDefault.yml' ) - assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.keySet(), - allOf( - containsInAnyOrder( - 'testStage1', - 'testStage3' - ), - hasSize(2) - ) - ) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(true)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage2, is(false)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage3, is(true)) assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true)) assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage2?.secondStep, is(false)) @@ -251,15 +232,9 @@ steps: {} stageConfigResource: 'testDefault.yml' ) - assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.keySet(), - allOf( - containsInAnyOrder( - 'testStage1', - 'testStage3' - ), - hasSize(2) - ) - ) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(true)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage2, is(false)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage3, is(true)) assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true)) assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage2?.secondStep, is(false)) @@ -308,6 +283,41 @@ steps: {} } + @Test + void testConditionFilePatternWithList() { + helper.registerAllowedMethod('libraryResource', [String.class], {s -> + if(s == 'testDefault.yml') { + return ''' +stages: + testStage1: + stepConditions: + firstStep: + filePattern: + - \'**/conf.js\' + - \'myCollection.json\' + secondStep: + filePattern: \'**/conf.jsx\' + +''' + } else { + return ''' +general: {} +steps: {} +''' + } + }) + + jsr.step.piperInitRunStageConfiguration( + script: nullScript, + juStabUtils: utils, + stageConfigResource: 'testDefault.yml' + ) + + assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.secondStep, is(false)) + + } + @Test void testConditionFilePatternFromConfig() { helper.registerAllowedMethod('libraryResource', [String.class], {s -> @@ -397,15 +407,8 @@ steps: {} stageConfigResource: 'com.sap.piper/pipeline/stageDefaults.yml' ) - assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.keySet(), - allOf( - containsInAnyOrder( - 'Acceptance', - 'Integration' - ), - hasSize(2) - ) - ) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.Acceptance, is(true)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.Integration, is(true)) } @@ -450,4 +453,229 @@ steps: {} assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.Acceptance, is(true)) } + + @Test + void testConditionNpmScripts() { + helper.registerAllowedMethod('libraryResource', [String.class], {s -> + if(s == 'testDefault.yml') { + return ''' +stages: + testStage1: + stepConditions: + firstStep: + npmScripts: \'npmScript\' + secondStep: + filePattern: \'**/conf.jsx\' + +''' + } else { + return ''' +general: {} +steps: {} +''' + } + }) + + helper.registerAllowedMethod('findFiles', [Map], {m -> + if(m.glob == '**/package.json') { + return [new File("package.json")].toArray() + } else { + return [] + } + }) + + helper.registerAllowedMethod('readJSON', [Map], { m -> + if (m.file == 'package.json') { + return [scripts: [npmScript: "echo test", + npmScript2: "echo test"]] + } else { + return [:] + } + }) + + jsr.step.piperInitRunStageConfiguration( + script: nullScript, + juStabUtils: utils, + stageConfigResource: 'testDefault.yml' + ) + + assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.secondStep, is(false)) + + } + + @Test + void testConditionNpmScriptsWithList() { + helper.registerAllowedMethod('libraryResource', [String.class], {s -> + if(s == 'testDefault.yml') { + return ''' +stages: + testStage1: + stepConditions: + firstStep: + npmScripts: + - \'npmScript\' + - \'npmScript2\' + secondStep: + filePattern: \'**/conf.jsx\' + +''' + } else { + return ''' +general: {} +steps: {} +''' + } + }) + + helper.registerAllowedMethod('findFiles', [Map], {m -> + if(m.glob == '**/package.json') { + return [new File("package.json")].toArray() + } else { + return [] + } + }) + + helper.registerAllowedMethod('readJSON', [Map], { m -> + if (m.file == 'package.json') { + return [scripts: [npmScript: "echo test", + npmScript2: "echo test"]] + } else { + return [:] + } + }) + + jsr.step.piperInitRunStageConfiguration( + script: nullScript, + juStabUtils: utils, + stageConfigResource: 'testDefault.yml' + ) + + assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.firstStep, is(true)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStep.testStage1.secondStep, is(false)) + + } + + @Test + void testConditionOnlyProductiveBranchOnNonProductiveBranch() { + helper.registerAllowedMethod('libraryResource', [String.class], {s -> + if(s == 'testDefault.yml') { + return ''' +stages: + testStage1: + onlyProductiveBranch: true + stepConditions: + firstStep: + filePattern: \'**/conf.js\' +''' + } else { + return ''' +general: {} +steps: {} +''' + } + }) + + binding.variables.env.BRANCH_NAME = 'test' + + jsr.step.piperInitRunStageConfiguration( + script: nullScript, + juStabUtils: utils, + stageConfigResource: 'testDefault.yml', + productiveBranch: 'master' + ) + + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(false)) + } + + @Test + void testConditionOnlyProductiveBranchOnProductiveBranch() { + helper.registerAllowedMethod('libraryResource', [String.class], {s -> + if(s == 'testDefault.yml') { + return ''' +stages: + testStage1: + onlyProductiveBranch: true + stepConditions: + firstStep: + filePattern: \'**/conf.js\' +''' + } else { + return ''' +general: {} +steps: {} +''' + } + }) + + binding.variables.env.BRANCH_NAME = 'test' + + jsr.step.piperInitRunStageConfiguration( + script: nullScript, + juStabUtils: utils, + stageConfigResource: 'testDefault.yml', + productiveBranch: 'test' + ) + + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(true)) + } + + @Test + void testStageExtensionExists() { + helper.registerAllowedMethod('libraryResource', [String.class], {s -> + if(s == 'testDefault.yml') { + return ''' +stages: + testStage1: + extensionExists: true + testStage2: + extensionExists: true + testStage3: + extensionExists: false + testStage4: + extensionExists: 'false' + testStage5: + dummy: true +''' + } else { + return ''' +general: + projectExtensionsDirectory: './extensions/' +steps: {} +''' + } + }) + + helper.registerAllowedMethod('fileExists', [String], {path -> + switch (path) { + case './extensions/testStage1.groovy': + return true + case './extensions/testStage2.groovy': + return false + case './extensions/testStage3.groovy': + return true + case './extensions/testStage4.groovy': + return true + case './extensions/testStage5.groovy': + return true + default: + return false + } + }) + + nullScript.piperStageWrapper = [:] + nullScript.piperStageWrapper.allowExtensions = {script -> return true} + + jsr.step.piperInitRunStageConfiguration( + script: nullScript, + juStabUtils: utils, + stageConfigResource: 'testDefault.yml' + ) + + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage1, is(true)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage2, is(false)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage3, is(false)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage4, is(false)) + assertThat(nullScript.commonPipelineEnvironment.configuration.runStage.testStage5, is(false)) + } } diff --git a/vars/piperInitRunStageConfiguration.groovy b/vars/piperInitRunStageConfiguration.groovy index b0e7a06d2..5b41f65be 100644 --- a/vars/piperInitRunStageConfiguration.groovy +++ b/vars/piperInitRunStageConfiguration.groovy @@ -13,7 +13,20 @@ import groovy.transform.Field * Print more detailed information into the log. * @possibleValues `true`, `false` */ - 'verbose' + 'verbose', + /** + * The branch used as productive branch, defaults to master. + */ + 'productiveBranch', + /** + * Location for individual stage extensions. + */ + 'projectExtensionsDirectory', + /** + * Location for global extensions. + */ + 'globalExtensionsDirectory' + ] @Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS.plus([ @@ -50,60 +63,52 @@ void call(Map parameters = [:]) { //handling of stage and step activation config.stages.each {stage -> - //activate stage if stage configuration is available - if (ConfigurationLoader.stageConfiguration(script, stage.getKey())) { - script.commonPipelineEnvironment.configuration.runStage[stage.getKey()] = true - } - //------------------------------------------------------------------------------- - //detailed handling of step and stage activation based on conditions - script.commonPipelineEnvironment.configuration.runStep[stage.getKey()] = [:] - def currentStage = stage.getKey() - stage.getValue().stepConditions.each {step -> - def stepActive = false + String currentStage = stage.getKey() + script.commonPipelineEnvironment.configuration.runStep[currentStage] = [:] + + // Always test step conditions in order to fill runStep[currentStage] map + boolean anyStepConditionTrue = false + stage.getValue().stepConditions?.each {step -> + boolean stepActive = false + String stepName = step.getKey() step.getValue().each {condition -> - Map stepConfig = script.commonPipelineEnvironment.getStepConfiguration(step.getKey(), currentStage) + Map stepConfig = script.commonPipelineEnvironment.getStepConfiguration(stepName, currentStage) switch(condition.getKey()) { case 'config': - if (condition.getValue() instanceof Map) { - condition.getValue().each {configCondition -> - if (MapUtils.getByPath(stepConfig, configCondition.getKey()) in configCondition.getValue()) { - stepActive = true - } - } - } else if (MapUtils.getByPath(stepConfig, condition.getValue())) { - stepActive = true - } + stepActive = stepActive || checkConfig(condition, stepConfig) break case 'configKeys': - if (condition.getValue() instanceof List) { - condition.getValue().each {configKey -> - if (MapUtils.getByPath(stepConfig, configKey)) { - stepActive = true - } - } - } else if (MapUtils.getByPath(stepConfig, condition.getValue())) { - stepActive = true - } + stepActive = stepActive || checkConfigKeys(condition, stepConfig) break case 'filePatternFromConfig': - def conditionValue = MapUtils.getByPath(stepConfig, condition.getValue()) - if (conditionValue && findFiles(glob: conditionValue)) { - stepActive = true - } + stepActive = stepActive || checkForFilesWithPatternFromConfig(script, condition, stepConfig) break case 'filePattern': - if (findFiles(glob: condition.getValue())) { - stepActive = true - } + stepActive = stepActive || checkForFilesWithPattern(script, condition) + break + case 'npmScripts': + stepActive = stepActive || checkForNpmScriptsInPackages(script, condition) break } } - script.commonPipelineEnvironment.configuration.runStep."${stage.getKey()}"."${step.getKey()}" = stepActive - - //make sure that also related stage is activated if steps are active - if (stepActive) script.commonPipelineEnvironment.configuration.runStage[stage.getKey()] = true + script.commonPipelineEnvironment.configuration.runStep[currentStage][stepName] = stepActive + anyStepConditionTrue = anyStepConditionTrue || stepActive } + + boolean runStage + if (stage.getValue().onlyProductiveBranch && (config.productiveBranch != env.BRANCH_NAME)) { + runStage = false + } else if (ConfigurationLoader.stageConfiguration(script, currentStage)) { + //activate stage if stage configuration is available + runStage = true + } else if (stage.getValue().extensionExists == true) { + runStage = anyStepConditionTrue || checkExtensionExists(script, config, currentStage) + } else { + runStage = anyStepConditionTrue + } + + script.commonPipelineEnvironment.configuration.runStage[currentStage] = runStage } if (config.verbose) { @@ -111,3 +116,92 @@ void call(Map parameters = [:]) { echo "[${STEP_NAME}] Debug - Run Step Configuration: ${script.commonPipelineEnvironment.configuration.runStep}" } } + +private static boolean checkExtensionExists(Script script, Map config, String stageName) { + if (!script.piperStageWrapper.allowExtensions(script)) { + return false + } + // NOTE: These keys exist in "config" if they are configured in the general section of the project + // config or the defaults. However, in piperStageWrapper, these keys could also be configured for + // the step "piperStageWrapper" to be effective. Don't know if this should be considered here for consistency. + if (!config.globalExtensionsDirectory && !config.projectExtensionsDirectory) { + return false + } + def projectInterceptorFile = "${config.projectExtensionsDirectory}${stageName}.groovy" + def globalInterceptorFile = "${config.globalExtensionsDirectory}${stageName}.groovy" + return script.fileExists(projectInterceptorFile) || script.fileExists(globalInterceptorFile) +} + +private static boolean checkConfig(def condition, Map stepConfig) { + Boolean configExists = false + if (condition.getValue() instanceof Map) { + condition.getValue().each {configCondition -> + if (MapUtils.getByPath(stepConfig, configCondition.getKey()) in configCondition.getValue()) { + configExists = true + } + } + } else if (MapUtils.getByPath(stepConfig, condition.getValue())) { + configExists = true + } + return configExists +} + +private static boolean checkConfigKeys(def condition, Map stepConfig) { + Boolean configKeyExists = false + if (condition.getValue() instanceof List) { + condition.getValue().each { configKey -> + if (MapUtils.getByPath(stepConfig, configKey)) { + configKeyExists = true + } + } + } else if (MapUtils.getByPath(stepConfig, condition.getValue())) { + configKeyExists = true + } + return configKeyExists +} + +private static boolean checkForFilesWithPatternFromConfig (Script script, def condition, Map stepConfig) { + def conditionValue = MapUtils.getByPath(stepConfig, condition.getValue()) + if (conditionValue && script.findFiles(glob: conditionValue)) { + return true + } + return false +} + +private static boolean checkForFilesWithPattern (Script script, def condition) { + Boolean filesExist = false + if (condition.getValue() instanceof List) { + condition.getValue().each {configKey -> + if (script.findFiles(glob: configKey)) { + filesExist = true + } + } + } else { + if (script.findFiles(glob: condition.getValue())) { + filesExist = true + } + } + return filesExist +} + +private static boolean checkForNpmScriptsInPackages (Script script, def condition) { + def packages = script.findFiles(glob: '**/package.json', excludes: '**/node_modules/**') + Boolean npmScriptExists = false + for (int i = 0; i < packages.size(); i++) { + String packageJsonPath = packages[i].path + Map packageJson = script.readJSON file: packageJsonPath + Map npmScripts = packageJson.scripts ?: [:] + if (condition.getValue() instanceof List) { + condition.getValue().each { configKey -> + if (npmScripts[configKey]) { + npmScriptExists = true + } + } + } else { + if (npmScripts[condition.getValue()]) { + npmScriptExists = true + } + } + } + return npmScriptExists +} diff --git a/vars/piperStageWrapper.groovy b/vars/piperStageWrapper.groovy index e1ff8ca57..548ff459f 100644 --- a/vars/piperStageWrapper.groovy +++ b/vars/piperStageWrapper.groovy @@ -109,7 +109,7 @@ private void executeStage(script, originalStage, stageName, config, utils, telem def body = originalStage // First, check if a global extension exists via a dedicated repository - if (globalExtensions && allowExtensions()) { + if (globalExtensions && allowExtensions(script)) { echo "[${STEP_NAME}] Found global interceptor '${globalInterceptorFile}' for ${stageName}." // If we call the global interceptor, we will pass on originalStage as parameter DebugReport.instance.globalExtensions.put(stageName, "Overwrites") @@ -124,7 +124,7 @@ private void executeStage(script, originalStage, stageName, config, utils, telem } // Second, check if a project extension (within the same repository) exists - if (projectExtensions && allowExtensions()) { + if (projectExtensions && allowExtensions(script)) { echo "[${STEP_NAME}] Running project interceptor '${projectInterceptorFile}' for ${stageName}." // If we call the project interceptor, we will pass on body as parameter which contains either originalStage or the repository interceptor if (projectExtensions && globalExtensions) { @@ -225,6 +225,6 @@ private boolean isOldInterceptorInterfaceUsed(Script interceptor) { return method != null } -private boolean allowExtensions() { - return env.PIPER_DISABLE_EXTENSIONS == null || Boolean.valueOf(env.PIPER_DISABLE_EXTENSIONS) == false +private static boolean allowExtensions(Script script) { + return script.env.PIPER_DISABLE_EXTENSIONS == null || Boolean.valueOf(script.env.PIPER_DISABLE_EXTENSIONS) == false }