Docker Pipeline - fixes and small adoptions (#779)

* buildExecute: Docker case - minor fixes * properly care for empty `containerBuildOptions` * verbose output wrt stashing * add stage ordinals * fix wrong script reference * fix null reference * adapt stage defaults and allow bats tests * propagate github org and repo * add test for bats
2025-01-20 05:19:40 +02:00 · 2019-07-05 16:32:24 +02:00 · 2019-07-05 16:32:24 +02:00 · 225cf2485b
commit 225cf2485b
parent 8f1b3e376c
13 changed files with 139 additions and 27 deletions
--- a/documentation/mkdocs.yml
+++ b/documentation/mkdocs.yml
@ -56,7 +56,7 @@ nav:
            - 'Stages':
                - 'Init Stage': stages/init.md
                - 'Pull-Request Voting Stage': stages/prvoting.md
-                - 'Central Build Stage': stages/build.md
+                - 'Build Stage': stages/build.md
                - 'Additional Unit Test Stage': stages/additionalunittests.md
                - 'Integration Stage': stages/integration.md
                - 'Acceptance Stage': stages/acceptance.md
--- a/resources/com.sap.piper/pipeline/stageDefaults.yml
+++ b/resources/com.sap.piper/pipeline/stageDefaults.yml
@ -2,27 +2,61 @@ stages:
  Init:
    stepConditions:
      slackSendNotification:
-        config: 'channel'
+        configKeys:
+          - 'channel'
  'Pull-Request Voting': {}
  Build: {}
-  'Additional Unit Tests': {}
+  'Additional Unit Tests':
+    stepConditions:
+      batsExecuteTests:
+        filePattern: '**/*.bats'
+      karmaExecuteTests:
+        filePattern: '**/karma.conf.js'
  Integration: {}
  Acceptance:
    stepConditions:
      cloudFoundryDeploy:
-        config: 'cfSpace'
+        configKeys:
+          - 'cfSpace'
+      healthExecuteCheck:
+        configKeys:
+          - 'testServerUrl'
      newmanExecute:
        filePatternFromConfig: 'newmanCollection'
-        config: 'testRepository'
+        configKeys:
+          - 'testRepository'
      uiVeri5ExecuteTests:
        filePattern: '**/conf.js'
-        config: 'testRepository'
-  Security: {}
+        configKeys:
+          - 'testRepository'
+  Security:
+    stepConditions:
+      whitesourceExecuteScan:
+        configKeys:
+          - 'userTokenCredentialsId'
+          - 'whitesource/userTokenCredentialsId'
+          - 'whitesourceUserTokenCredentialsId'
  Performance: {}
  Compliance: {}
-  Promote: {}
-  Release: {}
+  Promote:
+    stepConditions:
+      containerPushToRegistry:
+        configKeys:
+          - 'dockerRegistryUrl'
+  Release:
+    stepConditions:
+      cloudFoundryDeploy:
+        configKeys:
+          - 'cfSpace'
+          - 'cloudFoundry/space'
+      healthExecuteCheck:
+        configKeys:
+          - 'testServerUrl'
+      githubPublishRelease:
+        configKeys:
+          - 'githubTokenCredentialsId'
  'Post Actions':
    stepConditions:
      slackSendNotification:
-        config: 'channel'
+        configKeys:
+          - 'channel'
--- a/resources/com.sap.piper/pipeline/stageOrdinals.yml
+++ b/resources/com.sap.piper/pipeline/stageOrdinals.yml
@ -0,0 +1,23 @@
+stages:
+  Init:
+    ordinal: 1
+  'Pull-Request Voting':
+    ordinal: 5
+  Build:
+    ordinal: 10
+  'Additional Unit Tests':
+    ordinal: 20
+  Integration:
+    ordinal: 30
+  Acceptance:
+    ordinal: 40
+  Security:
+    ordinal: 50
+  Performance:
+    ordinal: 60
+  Compliance:
+    ordinal: 70
+  Promote:
+    ordinal: 80
+  Release:
+    ordinal: 90
--- a/test/groovy/BatsExecuteTestsTest.groovy
+++ b/test/groovy/BatsExecuteTestsTest.groovy
@ -64,8 +64,8 @@ class BatsExecuteTestsTest extends BasePiperTest {
        assertThat(dockerExecuteRule.dockerParams.dockerImage, is('node:8-stretch'))
        assertThat(dockerExecuteRule.dockerParams.dockerWorkspace, is('/home/node'))

-        assertThat(shellRule.shell, hasItem('npm install tap-xunit -g'))
-        assertThat(shellRule.shell, hasItem('cat \'TEST-testPackage.tap\' | tap-xunit --package=\'testPackage\' > TEST-testPackage.xml'))
+        assertThat(shellRule.shell, hasItem('NPM_CONFIG_PREFIX=~/.npm-global npm install tap-xunit -g'))
+        assertThat(shellRule.shell, hasItem('cat \'TEST-testPackage.tap\' | PATH=\$PATH:~/.npm-global/bin tap-xunit --package=\'testPackage\' > TEST-testPackage.xml'))

        assertJobStatusSuccess()
    }
--- a/test/groovy/templates/PiperPipelineStageAdditionalUnitTestsTest.groovy
+++ b/test/groovy/templates/PiperPipelineStageAdditionalUnitTestsTest.groovy
@ -39,6 +39,10 @@ class PiperPipelineStageAdditionalUnitTestsTest extends BasePiperTest {
            return body()
        })

+        helper.registerAllowedMethod('batsExecuteTests', [Map.class], {m ->
+            stepsCalled.add('batsExecuteTests')
+        })
+
        helper.registerAllowedMethod('karmaExecuteTests', [Map.class], {m ->
            stepsCalled.add('karmaExecuteTests')
        })
@ -53,7 +57,7 @@ class PiperPipelineStageAdditionalUnitTestsTest extends BasePiperTest {

        jsr.step.piperPipelineStageAdditionalUnitTests(script: nullScript, juStabUtils: utils)

-        assertThat(stepsCalled, not(hasItems('karmaExecuteTests', 'testsPublishResults')))
+        assertThat(stepsCalled, not(hasItems('batsExecuteTests', 'karmaExecuteTests', 'testsPublishResults')))
    }

    @Test
@ -65,4 +69,14 @@ class PiperPipelineStageAdditionalUnitTestsTest extends BasePiperTest {

        assertThat(stepsCalled, hasItems('karmaExecuteTests', 'testsPublishResults'))
    }
+
+    @Test
+    void testAdditionalUnitTestsWithBats() {
+
+        nullScript.commonPipelineEnvironment.configuration = [runStep: ['Additional Unit Tests': [batsExecuteTests: true]]]
+
+        jsr.step.piperPipelineStageAdditionalUnitTests(script: nullScript, juStabUtils: utils)
+
+        assertThat(stepsCalled, hasItems('batsExecuteTests', 'testsPublishResults'))
+    }
 }
--- a/test/groovy/templates/PiperPipelineStageInitTest.groovy
+++ b/test/groovy/templates/PiperPipelineStageInitTest.groovy
@ -139,17 +139,22 @@ class PiperPipelineStageInitTest extends BasePiperTest {
    void testSetScmInfoOnCommonPipelineEnvironment() {
        //currently supported formats
        def scmInfoTestList = [
-            [GIT_URL: 'https://github.com/testOrg/testRepo.git', expectedSsh: 'git@github.com:testOrg/testRepo.git', expectedHttp: 'https://github.com/testOrg/testRepo.git'],
-            [GIT_URL: 'https://github.com:7777/testOrg/testRepo.git', expectedSsh: 'git@github.com:testOrg/testRepo.git', expectedHttp: 'https://github.com:7777/testOrg/testRepo.git'],
-            [GIT_URL: 'git@github.com:testOrg/testRepo.git', expectedSsh: 'git@github.com:testOrg/testRepo.git', expectedHttp: 'https://github.com/testOrg/testRepo.git'],
-            [GIT_URL: 'ssh://git@github.com/testOrg/testRepo.git', expectedSsh: 'ssh://git@github.com/testOrg/testRepo.git', expectedHttp: 'https://github.com/testOrg/testRepo.git'],
-            [GIT_URL: 'ssh://git@github.com:7777/testOrg/testRepo.git', expectedSsh: 'ssh://git@github.com:7777/testOrg/testRepo.git', expectedHttp: 'https://github.com/testOrg/testRepo.git'],
+            [GIT_URL: 'https://github.com/testOrg/testRepo.git', expectedSsh: 'git@github.com:testOrg/testRepo.git', expectedHttp: 'https://github.com/testOrg/testRepo.git', expectedOrg: 'testOrg', expectedRepo: 'testRepo'],
+            [GIT_URL: 'https://github.com:7777/testOrg/testRepo.git', expectedSsh: 'git@github.com:testOrg/testRepo.git', expectedHttp: 'https://github.com:7777/testOrg/testRepo.git', expectedOrg: 'testOrg', expectedRepo: 'testRepo'],
+            [GIT_URL: 'git@github.com:testOrg/testRepo.git', expectedSsh: 'git@github.com:testOrg/testRepo.git', expectedHttp: 'https://github.com/testOrg/testRepo.git', expectedOrg: 'testOrg', expectedRepo: 'testRepo'],
+            [GIT_URL: 'ssh://git@github.com/testOrg/testRepo.git', expectedSsh: 'ssh://git@github.com/testOrg/testRepo.git', expectedHttp: 'https://github.com/testOrg/testRepo.git', expectedOrg: 'testOrg', expectedRepo: 'testRepo'],
+            [GIT_URL: 'ssh://git@github.com:7777/testOrg/testRepo.git', expectedSsh: 'ssh://git@github.com:7777/testOrg/testRepo.git', expectedHttp: 'https://github.com/testOrg/testRepo.git', expectedOrg: 'testOrg', expectedRepo: 'testRepo'],
+            [GIT_URL: 'ssh://git@github.com/path/to/testOrg/testRepo.git', expectedSsh: 'ssh://git@github.com/path/to/testOrg/testRepo.git', expectedHttp: 'https://github.com/path/to/testOrg/testRepo.git', expectedOrg: 'path/to/testOrg', expectedRepo: 'testRepo'],
+            [GIT_URL: 'ssh://git@github.com/testRepo.git', expectedSsh: 'ssh://git@github.com/testRepo.git', expectedHttp: 'https://github.com/testRepo.git', expectedOrg: 'N/A', expectedRepo: 'testRepo'],
        ]

        scmInfoTestList.each {scmInfoTest ->
            jsr.step.piperPipelineStageInit.setScmInfoOnCommonPipelineEnvironment(nullScript, scmInfoTest)
+            println(scmInfoTest.GIT_URL)
            assertThat(nullScript.commonPipelineEnvironment.getGitSshUrl(), is(scmInfoTest.expectedSsh))
            assertThat(nullScript.commonPipelineEnvironment.getGitHttpsUrl(), is(scmInfoTest.expectedHttp))
+            assertThat(nullScript.commonPipelineEnvironment.getGithubOrg(), is(scmInfoTest.expectedOrg))
+            assertThat(nullScript.commonPipelineEnvironment.getGithubRepo(), is(scmInfoTest.expectedRepo))
        }
    }

--- a/vars/batsExecuteTests.groovy
+++ b/vars/batsExecuteTests.groovy
@ -25,7 +25,7 @@ import groovy.transform.Field
    'failOnError',
    /**
     * Defines the format of the test result output. `junit` would be the standard for automated build environments but you could use also the option `tap`.
-     * @possibleValues `tap`
+     * @possibleValues `junit`, `tap`
     */
    'outputFormat',
    /**
@ -98,8 +98,8 @@ void call(Map parameters = [:]) {
                sh "cat 'TEST-${config.testPackage}.tap'"
                if (config.outputFormat == 'junit') {
                    dockerExecute(script: script, dockerImage: config.dockerImage, dockerWorkspace: config.dockerWorkspace, stashContent: config.stashContent) {
-                        sh "npm install tap-xunit -g"
-                        sh "cat 'TEST-${config.testPackage}.tap' | tap-xunit --package='${config.testPackage}' > TEST-${config.testPackage}.xml"
+                        sh "NPM_CONFIG_PREFIX=~/.npm-global npm install tap-xunit -g"
+                        sh "cat 'TEST-${config.testPackage}.tap' | PATH=\$PATH:~/.npm-global/bin tap-xunit --package='${config.testPackage}' > TEST-${config.testPackage}.xml"
                    }
                }
            }
--- a/vars/buildExecute.groovy
+++ b/vars/buildExecute.groovy
@ -93,10 +93,10 @@ void call(Map parameters = [:]) {
                    def containerImageNameAndTag = config.dockerRegistryUrl ? "${dockerUtils.getRegistryFromUrl(config.dockerRegistryUrl)}/${dockerImageNameAndTag}" : ''
                    kanikoExecute script: script, containerImageNameAndTag: containerImageNameAndTag
                } else {
-                    def dockerBuildImage = docker.build(dockerImageNameAndTag, "${config.containerBuildOptions} .")
+                    def dockerBuildImage = docker.build(dockerImageNameAndTag, "${config.containerBuildOptions ?: ''} .")
                    //only push if registry is defined
                    if (config.dockerRegistryUrl) {
-                        containerPushToRegistry script: this, dockerBuildImage: dockerBuildImage, dockerRegistryUrl: config.dockerRegistryUrl
+                        containerPushToRegistry script: script, dockerBuildImage: dockerBuildImage, dockerRegistryUrl: config.dockerRegistryUrl
                    }
                }
                script.commonPipelineEnvironment.setValue('containerImage', dockerImageNameAndTag)
--- a/vars/piperPipeline.groovy
+++ b/vars/piperPipeline.groovy
@ -12,7 +12,7 @@ void call(parameters) {
            stage('Init') {
                steps {
                    library 'piper-lib-os'
-                    piperPipelineStageInit script: parameters.script, customDefaults: parameters.customDefaults
+                    piperPipelineStageInit script: parameters.script, customDefaults: ['com.sap.piper/pipeline/stageOrdinals.yml'].plus(parameters.customDefaults ?: [])
                }
            }
            stage('Pull-Request Voting') {
--- a/vars/piperPipelineStageAdditionalUnitTests.groovy
+++ b/vars/piperPipelineStageAdditionalUnitTests.groovy
@ -9,7 +9,9 @@ import static com.sap.piper.Prerequisites.checkScript

@Field Set GENERAL_CONFIG_KEYS = []
@Field STAGE_STEP_KEYS = [
-    /** Executes karma tests which is for example suitable for OPA5 testing as well as QUnit testing of SAP UI5 apps.*/
+    /** Executes bats tests which are for example suitable for testing Docker images via a shell.*/
+    'batsExecuteTests',
+    /** Executes karma tests which are for example suitable for OPA5 testing as well as QUnit testing of SAP UI5 apps.*/
    'karmaExecuteTests',
    /** Publishes test results to Jenkins. It will automatically be active in cases tests are executed. */
    'testsPublishResults'
@ -34,6 +36,7 @@ void call(Map parameters = [:]) {
        .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS)
        .mixinStageConfig(script.commonPipelineEnvironment, stageName, STEP_CONFIG_KEYS)
        .mixin(parameters, PARAMETER_KEYS)
+        .addIfEmpty('batsExecuteTests', script.commonPipelineEnvironment.configuration.runStep?.get(stageName)?.batsExecuteTests)
        .addIfEmpty('karmaExecuteTests', script.commonPipelineEnvironment.configuration.runStep?.get(stageName)?.karmaExecuteTests)
        .use()

@ -42,6 +45,13 @@ void call(Map parameters = [:]) {
        // telemetry reporting
        utils.pushToSWA([step: STEP_NAME], config)

+        if (config.batsExecuteTests) {
+            durationMeasure(script: script, measurementName: 'bats_duration') {
+                batsExecuteTests script: script
+                testsPublishResults script: script
+            }
+        }
+
        if (config.karmaExecuteTests) {
            durationMeasure(script: script, measurementName: 'karma_duration') {
                karmaExecuteTests script: script
--- a/vars/piperPipelineStageBuild.groovy
+++ b/vars/piperPipelineStageBuild.groovy
@ -32,7 +32,7 @@ import static com.sap.piper.Prerequisites.checkScript
 * They type of build is defined using the configuration `buildTool`, see also step [buildExecute](../steps/buildExecute.md)
 *
 */
-@GenerateStageDocumentation(defaultStageName = 'Central Build')
+@GenerateStageDocumentation(defaultStageName = 'Build')
 void call(Map parameters = [:]) {

    def script = checkScript(this, parameters) ?: this
--- a/vars/piperPipelineStageInit.groovy
+++ b/vars/piperPipelineStageInit.groovy
@ -127,7 +127,7 @@ private void checkBuildTool(config) {

 private void initStashConfiguration (script, config) {
    Map stashConfiguration = readYaml(text: libraryResource(config.stashSettings))
-    echo "Stash config: stashConfiguration"
+    if (config.verbose) echo "Stash config: ${stashConfiguration}"
    script.commonPipelineEnvironment.configuration.stageStashes = stashConfiguration
 }

@ -135,11 +135,13 @@ private void setScmInfoOnCommonPipelineEnvironment(script, scmInfo) {

    def gitUrl = scmInfo.GIT_URL

+    def gitPath = ''
    if (gitUrl.startsWith('http')) {
        def httpPattern = /(https?):\/\/([^:\/]+)(?:[:\d\/]*)(.*)/
        def gitMatcher = gitUrl =~ httpPattern
        if (!gitMatcher.hasGroup() && gitMatcher.groupCount() != 3) return
        script.commonPipelineEnvironment.setGitSshUrl("git@${gitMatcher[0][2]}:${gitMatcher[0][3]}")
+        gitPath = gitMatcher[0][3]
        script.commonPipelineEnvironment.setGitHttpsUrl(gitUrl)
    } else if (gitUrl.startsWith('ssh')) {
        //(.*)@([^:\/]*)(?:[:\d\/]*)(.*)
@ -148,11 +150,33 @@ private void setScmInfoOnCommonPipelineEnvironment(script, scmInfo) {
        if (!gitMatcher.hasGroup() && gitMatcher.groupCount() != 3) return
        script.commonPipelineEnvironment.setGitSshUrl(gitUrl)
        script.commonPipelineEnvironment.setGitHttpsUrl("https://${gitMatcher[0][2]}/${gitMatcher[0][3]}")
+        gitPath = gitMatcher[0][3]
    }
    else if (gitUrl.indexOf('@') > 0) {
        script.commonPipelineEnvironment.setGitSshUrl(gitUrl)
+        gitPath = gitUrl.split(':')[1]
        script.commonPipelineEnvironment.setGitHttpsUrl("https://${(gitUrl.split('@')[1]).replace(':', '/')}")
    }
+
+    List gitPathParts = gitPath.split('/')
+    def gitFolder = 'N/A'
+    def gitRepo = 'N/A'
+    switch (gitPathParts.size()) {
+        case 1:
+            gitRepo = gitPathParts[0].replaceAll('.git', '')
+            break
+        case 2:
+            gitFolder = gitPathParts[0]
+            gitRepo = gitPathParts[1].replaceAll('.git', '')
+            break
+        case { it > 3 }:
+            gitRepo = gitPathParts[gitPathParts.size()-1].replaceAll('.git', '')
+            gitPathParts.remove(gitPathParts.size()-1)
+            gitFolder = gitPathParts.join('/')
+            break
+    }
+    script.commonPipelineEnvironment.setGithubOrg(gitFolder)
+    script.commonPipelineEnvironment.setGithubRepo(gitRepo)
 }

 private void setPullRequestStageStepActivation(script, config, List actions) {
--- a/vars/piperPipelineStageIntegration.groovy
+++ b/vars/piperPipelineStageIntegration.groovy
@ -1,4 +1,5 @@
 import com.sap.piper.ConfigurationHelper
+import com.sap.piper.GenerateStageDocumentation
 import com.sap.piper.Utils
 import groovy.transform.Field

@ -15,6 +16,7 @@ import static com.sap.piper.Prerequisites.checkScript
 * The stage allows to execute project-specific integration tests.<br />
 * Typically, integration tests are very project-specific, thus they can be defined here using the [stage extension mechanism](../extensibility.md).
 */
+@GenerateStageDocumentation(defaultStageName = 'Integration')
 void call(Map parameters = [:]) {

    def script = checkScript(this, parameters) ?: this