Add step for end to end test execution (#1786)

This change adds the end to end test execution capabilities of the SAP
Cloud SDK pipeline, i.e., the consumer can define a npm script which is
then executed using the npmExecuteScripts step.

The handling of generated results will be provided in a follow up PR.

Co-authored-by: Daniel Kurzynski <daniel.kurzynski@sap.com>

documentation/docs/steps/npmExecuteEndToEndTests.md

@@ -0,0 +1,7 @@
+# ${docGenStepName}
+
+## ${docGenDescription}
+
+## ${docGenParameters}
+
+## ${docGenConfiguration}

documentation/mkdocs.yml

@@ -81,6 +81,7 @@ nav:
         - neoDeploy: steps/neoDeploy.md
         - newmanExecute: steps/newmanExecute.md
         - nexusUpload: steps/nexusUpload.md
+        - npmExecuteEndToEndTests: steps/npmExecuteEndToEndTests.md
         - npmExecuteLint: steps/npmExecuteLint.md
         - npmExecuteScripts: steps/npmExecuteScripts.md
         - pipelineExecute: steps/pipelineExecute.md

test/groovy/CommonStepsTest.groovy

@@ -54,6 +54,7 @@ public class CommonStepsTest extends BasePiperTest{
             'piperExecuteBin',
             'piperPipeline',
             'prepareDefaultValues',
+            'runClosures',
             'setupCommonPipelineEnvironment'
         ]
 
@@ -118,6 +119,7 @@ public class CommonStepsTest extends BasePiperTest{
         'handlePipelineStepErrors', // special step (infrastructure)
         'piperStageWrapper', //intended to be called from within stages
         'buildSetResult',
+        'runClosures',
         'abapEnvironmentPullGitRepo', //implementing new golang pattern without fields
         'checkmarxExecuteScan', //implementing new golang pattern without fields
         'githubPublishRelease', //implementing new golang pattern without fields
@@ -207,7 +209,8 @@ public class CommonStepsTest extends BasePiperTest{
             'commonPipelineEnvironment',
             'piperPipeline',
             'piperExecuteBin',
-            'buildSetResult'
+            'buildSetResult',
+            'runClosures'
         ]
 
         def stepsWithWrongStepName = []

test/groovy/NpmExecuteEndToEndTestsTest.groovy

@@ -0,0 +1,280 @@
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.ExpectedException
+import org.junit.rules.RuleChain
+import util.BasePiperTest
+import util.JenkinsCredentialsRule
+import util.JenkinsMockStepRule
+import util.JenkinsReadYamlRule
+import util.JenkinsStepRule
+import util.Rules
+
+import static org.junit.Assert.assertFalse
+import static org.junit.Assert.assertTrue
+import static org.junit.Assert.assertTrue
+
+class NpmExecuteEndToEndTestsTest extends BasePiperTest {
+
+    private JenkinsStepRule stepRule = new JenkinsStepRule(this)
+    private ExpectedException thrown = ExpectedException.none()
+    private JenkinsMockStepRule npmExecuteScriptsRule = new JenkinsMockStepRule(this, 'npmExecuteScripts')
+    private JenkinsCredentialsRule credentialsRule = new JenkinsCredentialsRule(this)
+    private JenkinsReadYamlRule readYamlRule = new JenkinsReadYamlRule(this)
+
+    private boolean executedOnKubernetes = false
+    private boolean executedOnNode = false
+    private boolean executedInParallel = false
+
+    @Rule
+    public RuleChain ruleChain = Rules
+        .getCommonRules(this)
+        .around(thrown)
+        .around(readYamlRule)
+        .around(credentialsRule)
+        .around(stepRule)
+        .around(npmExecuteScriptsRule)
+
+    @Before
+    void init() {
+        helper.registerAllowedMethod("deleteDir", [], null)
+
+        helper.registerAllowedMethod('dockerExecuteOnKubernetes', [Map.class, Closure.class], {params, body ->
+            executedOnKubernetes = true
+            body()
+        })
+        helper.registerAllowedMethod('node', [String.class, Closure.class], {s, body ->
+            executedOnNode = true
+            body()
+        })
+        helper.registerAllowedMethod("parallel", [Map.class], { map ->
+            map.each {key, value ->
+                value()
+            }
+            executedInParallel = true
+        })
+
+        credentialsRule.reset()
+            .withCredentials('testCred', 'test_cf', '********')
+            .withCredentials('testCred2', 'test_other', '**')
+    }
+
+    @Test
+    void noAppUrl() {
+        thrown.expect(hudson.AbortException)
+        thrown.expectMessage('[npmExecuteEndToEndTests] The execution failed, since no appUrls are defined. Please provide appUrls as a list of maps.')
+
+        stepRule.step.npmExecuteEndToEndTests(
+            script: nullScript,
+            stage: "myStage",
+            runScript: "ci-e2e"
+        )
+    }
+
+    @Test
+    void noRunScript() {
+        def appUrl = [url: "http://my-url.com"]
+
+        nullScript.commonPipelineEnvironment.configuration = [stages: [myStage:[
+            appUrls: [appUrl]
+        ]]]
+
+        thrown.expect(hudson.AbortException)
+        thrown.expectMessage('[npmExecuteEndToEndTests] No runScript was defined.')
+
+        stepRule.step.npmExecuteEndToEndTests(
+            script: nullScript,
+            stage: "myStage"
+        )
+    }
+
+    @Test
+    void appUrlsNoList() {
+        def appUrl = "http://my-url.com"
+
+        nullScript.commonPipelineEnvironment.configuration = [stages: [myStage:[
+            appUrls: appUrl
+        ]]]
+
+        thrown.expect(hudson.AbortException)
+        thrown.expectMessage("[npmExecuteEndToEndTests] The execution failed, since appUrls is not a list. Please provide appUrls as a list of maps.")
+
+        stepRule.step.npmExecuteEndToEndTests(
+            script: nullScript,
+            stage: "myStage",
+            runScript: "ci-e2e"
+        )
+    }
+
+    @Test
+    void appUrlsNoMap() {
+        def appUrl = "http://my-url.com"
+
+        nullScript.commonPipelineEnvironment.configuration = [stages: [myStage:[
+            appUrls: [appUrl]
+        ]]]
+
+        thrown.expect(hudson.AbortException)
+        thrown.expectMessage("[npmExecuteEndToEndTests] The element ${appUrl} is not of type map. Please provide appUrls as a list of maps.")
+
+        stepRule.step.npmExecuteEndToEndTests(
+            script: nullScript,
+            stage: "myStage",
+            runScript: "ci-e2e"
+        )
+    }
+
+    @Test
+    void appUrlParametersNoList() {
+        def appUrl = [url: "http://my-url.com", credentialId: 'testCred', parameters: '--tag scenario1']
+
+        nullScript.commonPipelineEnvironment.configuration = [stages: [myStage:[
+            appUrls: [appUrl]
+        ]]]
+
+        thrown.expect(hudson.AbortException)
+        thrown.expectMessage("[npmExecuteEndToEndTests] The parameters property is not of type list. Please provide parameters as a list of strings.")
+
+        stepRule.step.npmExecuteEndToEndTests(
+            script: nullScript,
+            stage: "myStage",
+            runScript: "ci-e2e"
+        )
+    }
+
+    @Test
+    void oneAppUrl() {
+        def appUrl = [url: "http://my-url.com"]
+
+        nullScript.commonPipelineEnvironment.configuration = [stages: [myStage:[
+            appUrls: [appUrl]
+        ]]]
+
+        stepRule.step.npmExecuteEndToEndTests(
+            script: nullScript,
+            stage: "myStage",
+            runScript: "ci-e2e"
+        )
+
+        assertFalse(executedInParallel)
+        assert npmExecuteScriptsRule.hasParameter('script', nullScript)
+        assert npmExecuteScriptsRule.hasParameter('parameters', [dockerOptions: ['--shm-size 512MB']])
+        assert npmExecuteScriptsRule.hasParameter('install', false)
+        assert npmExecuteScriptsRule.hasParameter('virtualFrameBuffer', true)
+        assert npmExecuteScriptsRule.hasParameter('runScripts', ["ci-e2e"])
+        assert npmExecuteScriptsRule.hasParameter('scriptOptions', ["--launchUrl=${appUrl.url}"])
+    }
+
+    @Test
+    void oneAppUrlWithCredentials() {
+        def appUrl = [url: "http://my-url.com", credentialId: 'testCred']
+
+        nullScript.commonPipelineEnvironment.configuration = [stages: [myStage:[
+            appUrls: [appUrl]
+        ]]]
+
+        stepRule.step.npmExecuteEndToEndTests(
+            script: nullScript,
+            stage: "myStage",
+            runScript: "ci-e2e"
+        )
+
+        assert npmExecuteScriptsRule.hasParameter('script', nullScript)
+        assert npmExecuteScriptsRule.hasParameter('parameters', [dockerOptions: ['--shm-size 512MB']])
+        assert npmExecuteScriptsRule.hasParameter('install', false)
+        assert npmExecuteScriptsRule.hasParameter('virtualFrameBuffer', true)
+        assert npmExecuteScriptsRule.hasParameter('runScripts', ["ci-e2e"])
+        assert npmExecuteScriptsRule.hasParameter('scriptOptions', ["--launchUrl=${appUrl.url}"])
+    }
+
+    @Test
+    void twoAppUrlsWithCredentials() {
+        def appUrl = [url: "http://my-url.com", credentialId: 'testCred']
+        def appUrl2 = [url: "http://my-second-url.com", credentialId: 'testCred2']
+
+        nullScript.commonPipelineEnvironment.configuration = [stages: [myStage:[
+            appUrls: [appUrl, appUrl2]
+        ]]]
+
+        stepRule.step.npmExecuteEndToEndTests(
+            script: nullScript,
+            stage: "myStage",
+            runScript: "ci-e2e"
+        )
+
+        assert npmExecuteScriptsRule.hasParameter('script', nullScript)
+        assert npmExecuteScriptsRule.hasParameter('parameters', [dockerOptions: ['--shm-size 512MB']])
+        assert npmExecuteScriptsRule.hasParameter('install', false)
+        assert npmExecuteScriptsRule.hasParameter('virtualFrameBuffer', true)
+        assert npmExecuteScriptsRule.hasParameter('runScripts', ["ci-e2e"])
+        assert npmExecuteScriptsRule.hasParameter('scriptOptions', ["--launchUrl=${appUrl.url}"])
+        assert npmExecuteScriptsRule.hasParameter('scriptOptions', ["--launchUrl=${appUrl2.url}"])
+    }
+
+    @Test
+    void oneAppUrlWithCredentialsAndParameters() {
+        def appUrl = [url: "http://my-url.com", credentialId: 'testCred', parameters: ['--tag','scenario1', '--NIGHTWATCH_ENV=chrome']]
+
+        nullScript.commonPipelineEnvironment.configuration = [stages: [myStage:[
+            appUrls: [appUrl]
+        ]]]
+
+        stepRule.step.npmExecuteEndToEndTests(
+            script: nullScript,
+            stage: "myStage",
+            runScript: "ci-e2e"
+        )
+
+        assert npmExecuteScriptsRule.hasParameter('script', nullScript)
+        assert npmExecuteScriptsRule.hasParameter('parameters', [dockerOptions: ['--shm-size 512MB']])
+        assert npmExecuteScriptsRule.hasParameter('install', false)
+        assert npmExecuteScriptsRule.hasParameter('virtualFrameBuffer', true)
+        assert npmExecuteScriptsRule.hasParameter('runScripts', ["ci-e2e"])
+        assert npmExecuteScriptsRule.hasParameter('scriptOptions', ["--launchUrl=${appUrl.url}"] + appUrl.parameters)
+    }
+
+    @Test
+    void parallelE2eTest() {
+        def appUrl = [url: "http://my-url.com", credentialId: 'testCred']
+
+        nullScript.commonPipelineEnvironment.configuration = [
+            general: [parallelExecution: true],
+            stages: [
+                myStage:[
+                    appUrls: [appUrl]
+                ]]]
+
+        stepRule.step.npmExecuteEndToEndTests(
+            script: nullScript,
+            stage: "myStage",
+            runScript: "ci-e2e"
+        )
+
+        assertTrue(executedInParallel)
+        assertTrue(executedOnNode)
+        assertFalse(executedOnKubernetes)
+    }
+
+    @Test
+    void parallelE2eTestOnKubernetes() {
+        def appUrl = [url: "http://my-url.com", credentialId: 'testCred']
+        binding.variables.env.POD_NAME = "name"
+
+        nullScript.commonPipelineEnvironment.configuration = [
+            general: [parallelExecution: true],
+            stages: [
+                myStage:[
+                    appUrls: [appUrl]
+                ]]]
+
+        stepRule.step.npmExecuteEndToEndTests(
+            script: nullScript,
+            stage: "myStage",
+            runScript: "ci-e2e"
+        )
+
+        assertTrue(executedInParallel)
+        assertFalse(executedOnNode)
+        assertTrue(executedOnKubernetes)
+    }
+}

vars/multicloudDeploy.groovy

@@ -93,7 +93,7 @@ void call(parameters = [:]) {
                     )
                 }
             }
-            runClosures(config, createServices, "cloudFoundryCreateService")
+            runClosures(script, createServices, config.parallelExecution, "cloudFoundryCreateService")
         }
 
         if (config.cfTargets) {
@@ -176,21 +176,6 @@ void call(parameters = [:]) {
             error "Deployment skipped because no targets defined!"
         }
 
-        runClosures(config, deployments, "deployments")
-
-    }
-}
-
-def runClosures(Map config, Map toRun, String label = "closures") {
-    echo "Executing $label"
-    if (config.parallelExecution) {
-        echo "Executing $label in parallel"
-        parallel toRun
-    } else {
-        echo "Executing $label in sequence"
-        def closuresToRun = toRun.values().asList()
-        for (int i = 0; i < closuresToRun.size(); i++) {
-            (closuresToRun[i] as Closure)()
-        }
+        runClosures(script, deployments, config.parallelExecution, "deployments")
     }
 }

vars/npmExecuteEndToEndTests.groovy

@@ -0,0 +1,128 @@
+import com.sap.piper.ConfigurationHelper
+import com.sap.piper.DownloadCacheUtils
+import com.sap.piper.GenerateDocumentation
+import com.sap.piper.k8s.ContainerMap
+import groovy.transform.Field
+import com.sap.piper.Utils
+
+import static com.sap.piper.Prerequisites.checkScript
+
+@Field String STEP_NAME = getClass().getName()
+
+@Field Set GENERAL_CONFIG_KEYS = [
+    /** Executes the deployments in parallel.*/
+    'parallelExecution'
+]
+@Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS.plus([
+    /**
+     * The URLs under which the app is available after deployment.
+     * Each element of appUrls must be a map containing a property url, an optional property credentialId, and an optional property parameters.
+     * The optional property parameters can be used to pass additional parameters to the end-to-end test deployment reachable via the given application URL.
+     * These parameters must be a list of strings, where each string corresponds to one element of the parameters.
+     * For example, if the parameter `--tag scenario1` should be passed to the test, specify parameters: ["--tag", "scenario1"].
+     * These parameters are appended to the npm command during execution.
+     */
+    'appUrls',
+    /**
+     * Script to be executed from package.json.
+     */
+    'runScript'])
+@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS
+
+@Field Map CONFIG_KEY_COMPATIBILITY = [parallelExecution: 'features/parallelTestExecution']
+
+/**
+ * Executes end to end tests by running the npm script configured via the `runScript` property.
+ */
+@GenerateDocumentation
+void call(Map parameters = [:]) {
+    handlePipelineStepErrors(stepName: STEP_NAME, stepParameters: parameters) {
+        def script = checkScript(this, parameters) ?: this
+        def stageName = parameters.stage ?: env.STAGE_NAME
+
+        Map config = ConfigurationHelper.newInstance(this)
+            .loadStepDefaults()
+            .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY)
+            .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY)
+            .mixinStageConfig(script.commonPipelineEnvironment, stageName, STEP_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY)
+            .mixin(parameters, PARAMETER_KEYS)
+            .use()
+
+        // telemetry reporting
+        new Utils().pushToSWA([
+            step: STEP_NAME,
+            stepParamKey1: 'scriptMissing',
+            stepParam1: parameters?.script == null
+        ], config)
+
+        def e2ETests = [:]
+        def index = 1
+
+        def npmParameters = [:]
+        npmParameters.dockerOptions = ['--shm-size 512MB']
+
+        if (!config.appUrls) {
+            error "[${STEP_NAME}] The execution failed, since no appUrls are defined. Please provide appUrls as a list of maps.\n"
+
+        }
+        if (!(config.appUrls instanceof List)) {
+            error "[${STEP_NAME}] The execution failed, since appUrls is not a list. Please provide appUrls as a list of maps. For example:\n" +
+                "appUrls: \n" + "  - url: 'https://my-url.com'\n" + "    credentialId: myCreds"
+        }
+        if (!config.runScript) {
+            error "[${STEP_NAME}] No runScript was defined."
+        }
+
+        for (int i = 0; i < config.appUrls.size(); i++) {
+            List credentials = []
+            def appUrl = config.appUrls[i]
+
+            if (!(appUrl instanceof Map)) {
+                error "[${STEP_NAME}] The element ${appUrl} is not of type map. Please provide appUrls as a list of maps. For example:\n" +
+                    "appUrls: \n" + "  - url: 'https://my-url.com'\n" + "    credentialId: myCreds"
+            }
+            if (!appUrl.url) {
+                error "[${STEP_NAME}] No url property was defined for the following element in appUrls: ${appUrl}"
+            }
+            if (appUrl.credentialId) {
+                credentials.add(usernamePassword(credentialsId: appUrl.credentialId, passwordVariable: 'e2e_password', usernameVariable: 'e2e_username'))
+            }
+
+            Closure e2eTest = {
+                Utils utils = new Utils()
+                utils.unstashStageFiles(script, stageName)
+                try {
+                    withCredentials(credentials) {
+                        if (appUrl.parameters) {
+                            if (appUrl.parameters instanceof List) {
+                                npmExecuteScripts(script: script, parameters: npmParameters, install: false, virtualFrameBuffer: true, runScripts: [config.runScript], scriptOptions: ["--launchUrl=${appUrl.url}"] + appUrl.parameters)
+                            } else {
+                                error "[${STEP_NAME}] The parameters property is not of type list. Please provide parameters as a list of strings."
+                            }
+                        }
+                        npmExecuteScripts(script: script, parameters: npmParameters, install: false, virtualFrameBuffer: true, runScripts: [config.runScript], scriptOptions: ["--launchUrl=${appUrl.url}"])
+                    }
+
+                } catch (Exception e) {
+                    error "[${STEP_NAME}] The execution failed with error: ${e.getMessage()}"
+                } finally {
+                    //TODO: Implement Report handling
+                    utils.stashStageFiles(script, parameters.stage)
+                }
+            }
+            e2ETests["E2E Tests ${index > 1 ? index : ''}"] = {
+                if (env.POD_NAME) {
+                    dockerExecuteOnKubernetes(script: script, containerMap: ContainerMap.instance.getMap().get(parameters.stage) ?: [:]) {
+                        e2eTest.call()
+                    }
+                } else {
+                    node(env.NODE_NAME) {
+                        e2eTest.call()
+                    }
+                }
+            }
+            index++
+        }
+        runClosures(script, e2ETests, config.parallelExecution, "end to end tests")
+    }
+}

vars/runClosures.groovy

@@ -0,0 +1,15 @@
+void call(script, Map closures, Boolean parallelExecution, String label = "closures") {
+    handlePipelineStepErrors(stepName: 'runClosures', stepParameters: [script: script]) {
+        echo "Executing $label"
+        if (parallelExecution) {
+            echo "Executing $label in parallel"
+            parallel closures
+        } else {
+            echo "Executing $label in sequence"
+            def closuresToRun = closures.values().asList()
+            for (int i = 0; i < closuresToRun.size(); i++) {
+                (closuresToRun[i] as Closure)()
+            }
+        }
+    }
+}