add wrapper for stages contained in library (#341)

* add wrapper for stages contained in library

`piperStageWrapper` provides a wrapper for stages which we may include into the library.
It will take care about extension capabilities, locking, node handling, ... which should be a capability of every stage contained in the library.

resources/default_pipeline_environment.yml

@@ -252,7 +252,7 @@ steps:
       git: '**/gitmetadata/**'
       opa5: '**/*.*'
       opensourceConfiguration: '**/srcclr.yml, **/vulas-custom.properties, **/.nsprc, **/.retireignore, **/.retireignore.json, **/.snyk'
-      pipelineConfigAndTests: '.pipeline/*.*'
+      pipelineConfigAndTests: '.pipeline/**'
       securityDescriptor: '**/xs-security.json'
       tests: '**/pom.xml, **/*.json, **/*.xml, **/src/**, **/node_modules/**, **/specs/**, **/env/**, **/*.js'
     stashExcludes:
@@ -338,3 +338,12 @@ steps:
       developmentSystemId: null
   transportRequestUploadFile:
   transportRequestRelease:
+
+  #defaults for stage wrapper
+  piperStageWrapper:
+    projectExtensionsDirectory: '.pipeline/extensions/'
+    globalExtensionsDirectory: ''
+    stageLocking: true
+    nodeLabel: ''
+    stashContent:
+      - 'pipelineConfigAndTests'

src/com/sap/piper/Utils.groovy

@@ -22,6 +22,26 @@ def stash(name, include = '**/*.*', exclude = '') {
     steps.stash name: name, includes: include, excludes: exclude
 }
 
+def stashList(script, List stashes) {
+    for (def stash : stashes) {
+        def name = stash.name
+        def include = stash.includes
+        def exclude = stash.excludes
+
+        if (stash?.merge == true) {
+            String lockName = "${script.commonPipelineEnvironment.configuration.stashFiles}/${stash.name}"
+            lock(lockName) {
+                unstash stash.name
+                echo "Stash content: ${name} (include: ${include}, exclude: ${exclude})"
+                steps.stash name: name, includes: include, exclude: exclude, allowEmpty: true
+            }
+        } else {
+            echo "Stash content: ${name} (include: ${include}, exclude: ${exclude})"
+            steps.stash name: name, includes: include, exclude: exclude, allowEmpty: true
+        }
+    }
+}
+
 def stashWithMessage(name, msg, include = '**/*.*', exclude = '') {
     try {
         stash(name, include, exclude)
@@ -70,6 +90,7 @@ void pushToSWA(Map parameters, Map config) {
             custom3 = step name (passed as parameter step)
             custom4 = job url hashed (calculated)
             custom5 = build url hashed (calculated)
+            custom10 = stage name
             custom11 = step related parameter 1 (passed as parameter stepParam1)
             custom12 = step related parameter 2 (passed as parameter stepParam2)
             custom13 = step related parameter 3 (passed as parameter stepParam3)
@@ -81,11 +102,12 @@ void pushToSWA(Map parameters, Map config) {
         def action_name = 'Piper Library OS'
         def idsite = '827e8025-1e21-ae84-c3a3-3f62b70b0130'
         def url = 'https://github.com/SAP/jenkins-library'
-        def event_type = 'library-os'
+        def event_type = parameters.get('eventType') ?: 'library-os'
 
         swaCustom.custom3 = parameters.get('step')
         swaCustom.custom4 = generateSha1Inline(env.JOB_URL)
         swaCustom.custom5 = generateSha1Inline(env.BUILD_URL)
+        swaCustom.custom10 = parameters.get('stageName')
         swaCustom.custom11 = parameters.get('stepParam1')
         swaCustom.custom12 = parameters.get('stepParam2')
         swaCustom.custom13 = parameters.get('stepParam3')
@@ -99,7 +121,7 @@ void pushToSWA(Map parameters, Map config) {
         options.push("--data-urlencode \"idsite=${idsite}\"")
         options.push("--data-urlencode \"url=${url}\"")
         options.push("--data-urlencode \"event_type=${event_type}\"")
-        for(def key : ['custom3', 'custom4', 'custom5', 'custom11', 'custom12', 'custom13', 'custom14', 'custom15']){
+        for(def key : ['custom3', 'custom4', 'custom5', 'custom10', 'custom11', 'custom12', 'custom13', 'custom14', 'custom15']){
             if (swaCustom[key] != null) options.push("--data-urlencode \"${key}=${swaCustom[key]}\"")
         }
         options.push("--connect-timeout 5")

test/groovy/CommonStepsTest.groovy

@@ -109,6 +109,7 @@ public class CommonStepsTest extends BasePiperTest{
             'pipelineExecute', // special step (infrastructure)
             'commonPipelineEnvironment', // special step (infrastructure)
             'handlePipelineStepErrors', // special step (infrastructure)
+            'piperStageWrapper' //intended to be called from within stages
             ]
 
     @Test

test/groovy/PiperStageWrapperTest.groovy

@@ -0,0 +1,118 @@
+#!groovy
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.RuleChain
+import util.BasePiperTest
+import util.JenkinsLoggingRule
+import util.JenkinsReadYamlRule
+import util.JenkinsStepRule
+import util.Rules
+
+import static org.hamcrest.CoreMatchers.containsString
+import static org.hamcrest.CoreMatchers.is
+import static org.junit.Assert.assertThat
+
+class PiperStageWrapperTest extends BasePiperTest {
+
+    private JenkinsLoggingRule jlr = new JenkinsLoggingRule(this)
+    private JenkinsStepRule jsr = new JenkinsStepRule(this)
+
+    private Map lockMap = [:]
+    private int countNodeUsage = 0
+    private String nodeLabel = ''
+
+    @Rule
+    public RuleChain rules = Rules
+        .getCommonRules(this)
+        .around(new JenkinsReadYamlRule(this))
+        .around(jlr)
+        .around(jsr)
+
+    @Before
+    void init() throws Exception {
+
+        helper.registerAllowedMethod('deleteDir', [], {return null})
+        helper.registerAllowedMethod('lock', [Map.class, Closure.class], {m, body ->
+            assertThat(m.resource.toString(), containsString('/10'))
+            lockMap = m
+            body()
+        })
+        helper.registerAllowedMethod('milestone', [Integer.class], {ordinal ->
+            assertThat(ordinal, is(10))
+        })
+        helper.registerAllowedMethod('node', [String.class, Closure.class], {s, body ->
+            nodeLabel = s
+            countNodeUsage++
+            body()
+
+        })
+        helper.registerAllowedMethod('fileExists', [String.class], {s ->
+            return false
+        })
+    }
+
+    @Test
+    void testDefault() {
+        def testInt = 1
+        jsr.step.piperStageWrapper(
+            script: nullScript,
+            juStabUtils: utils,
+            ordinal: 10,
+            stageName: 'test'
+
+        ) {
+            testInt ++
+        }
+        assertThat(testInt, is(2))
+        assertThat(lockMap.size(), is(2))
+        assertThat(countNodeUsage, is(1))
+    }
+
+    @Test
+    void testNoLocking() {
+        def testInt = 1
+        jsr.step.piperStageWrapper(
+            script: nullScript,
+            juStabUtils: utils,
+            nodeLabel: 'testLabel',
+            ordinal: 10,
+            stageLocking: false,
+            stageName: 'test'
+
+        ) {
+            testInt ++
+        }
+        assertThat(testInt, is(2))
+        assertThat(lockMap.size(), is(0))
+        assertThat(countNodeUsage, is(1))
+        assertThat(nodeLabel, is('testLabel'))
+    }
+
+    @Test
+    void testStageExit() {
+        helper.registerAllowedMethod('fileExists', [String.class], {s ->
+            return (s == '.pipeline/extensions/test.groovy')
+        })
+
+        helper.registerAllowedMethod('load', [String.class], {
+            return helper.loadScript('test/resources/stages/test.groovy')
+        })
+
+        def testInt = 1
+        jsr.step.piperStageWrapper(
+            script: nullScript,
+            juStabUtils: utils,
+            ordinal: 10,
+            stageName: 'test'
+        ) {
+            testInt ++
+        }
+
+        assertThat(testInt, is(2))
+        assertThat(jlr.log, containsString('[piperStageWrapper] Running project interceptor \'.pipeline/extensions/test.groovy\' for test.'))
+        assertThat(jlr.log, containsString('Stage Name: test'))
+        assertThat(jlr.log, containsString('Config:'))
+    }
+}
+

test/groovy/util/LibraryLoadingTestExecutionListener.groovy

@@ -123,7 +123,7 @@ class LibraryLoadingTestExecutionListener extends AbstractTestExecutionListener
     void beforeTestMethod(TestContext testContext) throws Exception {
         super.beforeTestMethod(testContext)
         def testInstance = testContext.getTestInstance()
-        testInstance.binding.setVariable('currentBuild', [result: 'SUCCESS'])
+        testInstance.binding.setVariable('currentBuild', [result: 'SUCCESS', currentResult: 'SUCCESS'])
         PipelineTestHelper helper = LibraryLoadingTestExecutionListener.getSingletonInstance()
         LibraryLoadingTestExecutionListener.START_METHOD_TRACKING = true
     }

test/resources/stages/test.groovy

@@ -0,0 +1,6 @@
+void call(body, stageName, config) {
+    echo "Stage Name: ${stageName}"
+    echo "Config: ${config}"
+    body()
+}
+return this

vars/piperStageWrapper.groovy

@@ -0,0 +1,108 @@
+import com.sap.piper.Utils
+import com.sap.piper.ConfigurationHelper
+import com.sap.piper.ConfigurationLoader
+import com.sap.piper.k8s.ContainerMap
+import groovy.transform.Field
+
+import static com.sap.piper.Prerequisites.checkScript
+
+@Field String STEP_NAME = 'piperStageWrapper'
+
+void call(Map parameters = [:], body) {
+
+    final script = checkScript(this, parameters) ?: this
+    def utils = parameters.juStabUtils ?: new Utils()
+
+    def stageName = parameters.stageName?:env.STAGE_NAME
+
+    // load default & individual configuration
+    Map config = ConfigurationHelper.newInstance(this)
+        .loadStepDefaults()
+        .mixin(ConfigurationLoader.defaultStageConfiguration(this, stageName))
+        .mixinGeneralConfig(script.commonPipelineEnvironment)
+        .mixinStageConfig(script.commonPipelineEnvironment, stageName)
+        .mixin(parameters)
+        .addIfEmpty('stageName', stageName)
+        .dependingOn('stageName').mixin('ordinal')
+        .use()
+
+    stageLocking(config) {
+        def containerMap = ContainerMap.instance.getMap().get(stageName) ?: [:]
+        if (Boolean.valueOf(env.ON_K8S) && containerMap.size() > 0) {
+            withEnv(["POD_NAME=${stageName}"]) {
+                dockerExecuteOnKubernetes(script: script, containerMap: containerMap) {
+                    executeStage(script, body, stageName, config, utils)
+                }
+            }
+        } else {
+            node(config.nodeLabel) {
+                executeStage(script, body, stageName, config, utils)
+            }
+        }
+    }
+}
+
+private void stageLocking(Map config, Closure body) {
+    if (config.stageLocking) {
+        lock(resource: "${env.JOB_NAME}/${config.ordinal}", inversePrecedence: true) {
+            milestone config.ordinal
+            body()
+        }
+    } else {
+        body()
+    }
+}
+
+private void executeStage(script, originalStage, stageName, config, utils) {
+
+    boolean projectExtensions
+    boolean globalExtensions
+    def startTime = System.currentTimeMillis()
+
+    try {
+        //Add general stage stashes to config.stashContent
+        config.stashContent += script.commonPipelineEnvironment.configuration.stageStashes?.get(stageName)?.unstash ?: []
+
+        utils.unstashAll(config.stashContent)
+
+        /* Defining the sources where to look for a project extension and a repository extension.
+        * Files need to be named like the executed stage to be recognized.
+        */
+        def projectInterceptorFile = "${config.projectExtensionsDirectory}${stageName}.groovy"
+        def globalInterceptorFile = "${config.globalExtensionsDirectory}${stageName}.groovy"
+        projectExtensions = fileExists(projectInterceptorFile)
+        globalExtensions = fileExists(globalInterceptorFile)
+        // Pre-defining the real originalStage in body variable, might be overwritten later if extensions exist
+        def body = originalStage
+
+        // First, check if a global extension exists via a dedicated repository
+        if (globalExtensions) {
+            Script globalInterceptorScript = load(globalInterceptorFile)
+            echo "[${STEP_NAME}] Found global interceptor '${globalInterceptorFile}' for ${stageName}."
+            // If we call the global interceptor, we will pass on originalStage as parameter
+            body = {
+                globalInterceptorScript(body, stageName, config)
+            }
+        }
+
+        // Second, check if a project extension (within the same repository) exists
+        if (projectExtensions) {
+            Script projectInterceptorScript = load(projectInterceptorFile)
+            echo "[${STEP_NAME}] Running project interceptor '${projectInterceptorFile}' for ${stageName}."
+            // If we call the project interceptor, we will pass on body as parameter which contains either originalStage or the repository interceptor
+            projectInterceptorScript(body, stageName, config)
+        } else {
+            //TODO: assign projectInterceptorScript to body as done for globalInterceptorScript, currently test framework does not seem to support this case. Further investigations needed.
+            body()
+        }
+
+    } finally {
+        echo "Current build result in stage $stageName is ${script.currentBuild.currentResult}."
+        //Perform stashing of selected files in workspace
+        utils.stashList(script, script.commonPipelineEnvironment.configuration.stageStashes?.get(stageName)?.stashes ?: [])
+        deleteDir()
+
+        def duration = System.currentTimeMillis() - startTime
+        utils.pushToSWA([eventType: 'library-os-stage', stageName: stageName, stepParam1: "${script.currentBuild.currentResult}", stepParam2: "${startTime}", stepParam3: "${duration}", stepParam4: "${projectExtensions}", stepParam5: "${globalExtensions}"], config)
+    }
+}