Merge branch 'master' into SarahNoack-patch-2

documentation/bin/createDocu.groovy

@@ -418,6 +418,19 @@ for (step in steps) {
     }
 }
 
+// replace @see tag in docu by docu from referenced step.
+for(step in stepDescriptors) {
+    if(step.value.parameters) {
+        for(param in step.value.parameters) {
+            if( param?.value?.docu?.contains('@see')) {
+                def otherStep = param.value.docu.replaceAll('@see', '').trim()
+                param.value.docu = fetchTextFrom(otherStep, param.key, stepDescriptors)
+                param.value.mandatory = fetchMandatoryFrom(otherStep, param.key, stepDescriptors)
+            }
+        }
+    }
+}
+
 for(step in stepDescriptors) {
     try {
         renderStep(step.key, step.value)
@@ -461,6 +474,26 @@ void renderStep(stepName, stepProperties) {
     theStepDocu.withWriter { w -> w.write text }
 }
 
+def fetchTextFrom(def step, def parameterName, def steps) {
+    try {
+        def docuFromOtherStep = steps[step]?.parameters[parameterName]?.docu
+        if(! docuFromOtherStep) throw new IllegalStateException("No docu found for parameter '${parameterName}' in step ${step}.")
+        return docuFromOtherStep
+    } catch(e) {
+        System.err << "[ERROR] Cannot retrieve docu for parameter ${parameterName} from step ${step}.\n"
+        throw e
+    }
+}
+
+def fetchMandatoryFrom(def step, def parameterName, def steps) {
+    try {
+        return steps[step]?.parameters[parameterName]?.mandatory
+    } catch(e) {
+        System.err << "[ERROR] Cannot retrieve docu for parameter ${parameterName} from step ${step}.\n"
+        throw e
+    }
+}
+
 def handleStep(stepName, prepareDefaultValuesStep, gse) {
 
     File theStep = new File(stepsDir, "${stepName}.groovy")

documentation/docs/steps/dockerExecuteOnKubernetes.md

@@ -19,6 +19,7 @@ Executes a closure inside a container in a kubernetes pod. Proxy environment var
 |containerCommand|no|||
 |containerCommands|no|||
 |containerEnvVars|no|||
+|containerPullImageFlags|no|true|boolean value: `true`, `false` |
 |containerMap|no|`[:]`||
 |containerName|no|||
 |containerPortMappings|no|||
@@ -26,6 +27,7 @@ Executes a closure inside a container in a kubernetes pod. Proxy environment var
 |containerWorkspaces|no|||
 |dockerEnvVars|no|`[:]`||
 |dockerImage|yes|||
+|dockerPullImage|no|true|boolean value: `true`, `false` |
 |dockerWorkspace|no|`''`||
 |jenkinsKubernetes|no|`[jnlpAgent:s4sdk/jenkins-agent-k8s:latest]`||
 |stashExcludes|no|`[workspace:nohup.out]`||
@@ -35,6 +37,7 @@ Executes a closure inside a container in a kubernetes pod. Proxy environment var
 * `containerCommand`: allows to specify start command for container created with dockerImage parameter to overwrite Piper default (`/usr/bin/tail -f /dev/null`).
 * `containerCommands` specifies start command for containers to overwrite Piper default (`/usr/bin/tail -f /dev/null`). If container's defaultstart command should be used provide empty string like: `['selenium/standalone-chrome': '']`.
 * `containerEnvVars` specifies environment variables per container. If not provided `dockerEnvVars` will be used.
+* `containerPullImageFlags` specifies the pullImage flag per container.
 * `containerMap` A map of docker image to the name of the container. The pod will be created with all the images from this map and they are labled based on the value field of each map entry.
    Example: `['maven:3.5-jdk-8-alpine': 'mavenExecute', 'selenium/standalone-chrome': 'selenium', 'famiko/jmeter-base': 'checkJMeter', 's4sdk/docker-cf-cli': 'cloudfoundry']`
 * `containerName`: optional configuration in combination with containerMap to define the container where the commands should be executed in
@@ -43,6 +46,7 @@ Executes a closure inside a container in a kubernetes pod. Proxy environment var
 * `containerWorkspaces` specifies workspace (=home directory of user) per container. If not provided `dockerWorkspace` will be used. If empty, home directory will not be set.
 * `dockerImage` Name of the docker image that should be used. If empty, Docker is not used.
 * `dockerEnvVars` Environment variables to set in the container, e.g. [http_proxy:'proxy:8080']
+* `dockerPullImage`: Set this to 'false' to bypass a docker image pull. Usefull during development process. Allows testing of images which are available in the local registry only.
 * `dockerWorkspace` Docker options to be set when starting the container. It can be a list or a string.
 
 ## Step configuration
@@ -56,12 +60,14 @@ In following sections the configuration is possible:
 |script||||
 |containerCommands||X|X|
 |containerEnvVars||X|X|
+|containerPullImageFlags||X|X|
 |containerMap||X|X|
 |containerName||X|X|
 |containerPortMappings||X|X|
 |containerWorkspaces||X|X|
 |dockerEnvVars||X|X|
 |dockerImage||X|X|
+|dockerPullImage||X|X|
 |dockerWorkspace||X|X|
 |jenkinsKubernetes|X|||
 |stashExcludes||X|X|

resources/default_pipeline_environment.yml

@@ -154,8 +154,11 @@ steps:
       - 'tests'
     testReportFilePath: 'cst-report.json'
   dockerExecute:
+    dockerPullImage: true
+    sidecarPullImage: true
     stashContent: []
   dockerExecuteOnKubernetes:
+    dockerPullImage: true
     stashContent: []
     stashIncludes:
       workspace: '**/*'

test/groovy/DockerExecuteOnKubernetesTest.groovy

@@ -53,6 +53,7 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
     def envList = []
     def portList = []
     def containerCommands = []
+    def pullImageMap = [:]
 
 
     @Before
@@ -78,6 +79,7 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
                 if (option.command) {
                     containerCommands.add(option.command)
                 }
+                pullImageMap.put(option.image.toString(), option.alwaysPullImage)
             }
             body()
         })
@@ -287,6 +289,50 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
         assertThat(containerCommands, hasItem('/busybox/tail -f /dev/null'))
     }
 
+    @Test
+    void testSkipDockerImagePull() throws Exception {
+        stepRule.step.dockerExecuteOnKubernetes(
+            script: nullScript,
+            dockerPullImage: false,
+            containerMap: ['maven:3.5-jdk-8-alpine': 'mavenexecute']
+        ) {
+            container(name: 'mavenexecute') {
+                bodyExecuted = true
+            }
+        }
+        assertEquals(false, pullImageMap.get('maven:3.5-jdk-8-alpine'))
+        assertTrue(bodyExecuted)
+    }
+
+    @Test
+    void testSkipSidecarImagePull() throws Exception {
+        stepRule.step.dockerExecuteOnKubernetes(
+            script: nullScript,
+            juStabUtils: utils,
+            containerCommands: ['selenium/standalone-chrome': ''],
+            containerEnvVars: [
+                'selenium/standalone-chrome': ['customEnvKey': 'customEnvValue']
+            ],
+            containerMap: [
+                'maven:3.5-jdk-8-alpine': 'mavenexecute',
+                'selenium/standalone-chrome': 'selenium'
+            ],
+            containerName: 'mavenexecute',
+            containerWorkspaces: [
+                'selenium/standalone-chrome': ''
+            ],
+            containerPullImageFlags: [
+                'maven:3.5-jdk-8-alpine': true,
+                'selenium/standalone-chrome': false
+            ],
+            dockerWorkspace: '/home/piper'
+        ) {
+            bodyExecuted = true
+        }
+        assertEquals(true, pullImageMap.get('maven:3.5-jdk-8-alpine'))
+        assertEquals(false, pullImageMap.get('selenium/standalone-chrome'))
+        assertTrue(bodyExecuted)
+    }
 
     private container(options, body) {
         containerName = options.name

test/groovy/DockerExecuteTest.groovy

@@ -137,6 +137,38 @@ class DockerExecuteTest extends BasePiperTest {
         assertTrue(bodyExecuted)
     }
 
+    @Test
+    void testSkipDockerImagePull() throws Exception {
+        nullScript.commonPipelineEnvironment.configuration = [steps:[dockerExecute:[dockerPullImage: false]]]
+        stepRule.step.dockerExecute(
+            script: nullScript,
+            dockerImage: 'maven:3.5-jdk-8-alpine'
+        ) {
+            bodyExecuted = true
+        }
+        assertThat(docker.imagePullCount, is(0))
+        assertThat(bodyExecuted, is(true))
+    }
+
+    @Test
+    void testSkipSidecarImagePull() throws Exception {
+        stepRule.step.dockerExecute(
+            script: nullScript,
+            dockerName: 'maven',
+            dockerImage: 'maven:3.5-jdk-8-alpine',
+            sidecarEnvVars: ['testEnv':'testVal'],
+            sidecarImage: 'selenium/standalone-chrome',
+            sidecarVolumeBind: ['/dev/shm':'/dev/shm'],
+            sidecarName: 'testAlias',
+            sidecarPorts: ['4444':'4444', '1111':'1111'],
+            sidecarPullImage: false
+        ) {
+            bodyExecuted = true
+        }
+        assertThat(docker.imagePullCount, is(1))
+        assertThat(bodyExecuted, is(true))
+    }
+
     @Test
     void testExecuteInsideDockerContainerWithParameters() throws Exception {
         stepRule.step.dockerExecute(script: nullScript,

vars/dockerExecute.groovy

@@ -1,13 +1,11 @@
 import static com.sap.piper.Prerequisites.checkScript
 
 import com.cloudbees.groovy.cps.NonCPS
-
 import com.sap.piper.ConfigurationHelper
 import com.sap.piper.GenerateDocumentation
 import com.sap.piper.JenkinsUtils
 import com.sap.piper.Utils
 import com.sap.piper.k8s.ContainerMap
-
 import groovy.transform.Field
 
 @Field def STEP_NAME = getClass().getName()
@@ -57,6 +55,10 @@ import groovy.transform.Field
      * Volumes that should be mounted into the container.
      */
     'dockerVolumeBind',
+    /**
+     * Set this to 'false' to bypass a docker image pull. Usefull during development process. Allows testing of images which are available in the local registry only.
+     */
+    'dockerPullImage',
     /**
      * Kubernetes only:
      * Specifies a dedicated user home directory for the container which will be passed as value for environment variable `HOME`.
@@ -82,6 +84,10 @@ import groovy.transform.Field
      * as `dockerVolumeBind` for the sidecar container
      */
     'sidecarVolumeBind',
+    /**
+     * Set this to 'false' to bypass a docker image pull. Usefull during development process. Allows testing of images which are available in the local registry only.
+     */
+    'sidecarPullImage',
     /**
      * as `dockerWorkspace` for the sidecar container
      */
@@ -127,6 +133,7 @@ void call(Map parameters = [:], body) {
                         containerCommand: config.containerCommand,
                         containerShell: config.containerShell,
                         dockerImage: config.dockerImage,
+                        dockerPullImage: config.dockerPullImage,
                         dockerEnvVars: config.dockerEnvVars,
                         dockerWorkspace: config.dockerWorkspace,
                         stashContent: config.stashContent
@@ -139,6 +146,7 @@ void call(Map parameters = [:], body) {
                         script: script,
                         containerCommands: [:],
                         containerEnvVars: [:],
+                        containerPullImageFlags: [:],
                         containerMap: [:],
                         containerName: config.dockerName,
                         containerPortMappings: [:],
@@ -150,6 +158,9 @@ void call(Map parameters = [:], body) {
                     paramMap.containerEnvVars[config.dockerImage] = config.dockerEnvVars
                     paramMap.containerEnvVars[config.sidecarImage] = config.sidecarEnvVars
 
+                    paramMap.containerPullImageFlags[config.dockerImage] = config.dockerPullImage
+                    paramMap.containerPullImageFlags[config.sidecarImage] = config.sidecarPullImage
+
                     paramMap.containerMap[config.dockerImage] = config.dockerName
                     paramMap.containerMap[config.sidecarImage] = config.sidecarName
 
@@ -179,7 +190,8 @@ void call(Map parameters = [:], body) {
             if (executeInsideDocker && config.dockerImage) {
                 utils.unstashAll(config.stashContent)
                 def image = docker.image(config.dockerImage)
-                image.pull()
+                if (config.dockerPullImage) image.pull()
+                else echo"[INFO][$STEP_NAME] Skipped pull of image '${config.dockerImage}'."
                 if (!config.sidecarImage) {
                     image.inside(getDockerOptions(config.dockerEnvVars, config.dockerVolumeBind, config.dockerOptions)) {
                         body()
@@ -189,14 +201,15 @@ void call(Map parameters = [:], body) {
                     sh "docker network create ${networkName}"
                     try{
                         def sidecarImage = docker.image(config.sidecarImage)
-                        sidecarImage.pull()
+                        if (config.sidecarPullImage) sidecarImage.pull()
+                        else echo"[INFO][$STEP_NAME] Skipped pull of image '${config.sidecarImage}'."
                         config.sidecarOptions = config.sidecarOptions?:[]
-                        if(config.sidecarName)
+                        if (config.sidecarName)
                             config.sidecarOptions.add("--network-alias ${config.sidecarName}")
                         config.sidecarOptions.add("--network ${networkName}")
                         sidecarImage.withRun(getDockerOptions(config.sidecarEnvVars, config.sidecarVolumeBind, config.sidecarOptions)) { c ->
                             config.dockerOptions = config.dockerOptions?:[]
-                            if(config.dockerName)
+                            if (config.dockerName)
                                 config.dockerOptions.add("--network-alias ${config.dockerName}")
                             config.dockerOptions.add("--network ${networkName}")
                             image.inside(getDockerOptions(config.dockerEnvVars, config.dockerVolumeBind, config.dockerOptions)) {

vars/dockerExecuteOnKubernetes.groovy

@@ -16,12 +16,14 @@ import hudson.AbortException
     'containerCommand', // specify start command for container created with dockerImage parameter to overwrite Piper default (`/usr/bin/tail -f /dev/null`).
     'containerCommands', //specify start command for containers to overwrite Piper default (`/usr/bin/tail -f /dev/null`). If container's default start command should be used provide empty string like: `['selenium/standalone-chrome': '']`
     'containerEnvVars', //specify environment variables per container. If not provided dockerEnvVars will be used
+    'containerPullImageFlags', // specifies the pullImage flag per container.
     'containerMap', //specify multiple images which then form a kubernetes pod, example: containerMap: ['maven:3.5-jdk-8-alpine': 'mavenexecute','selenium/standalone-chrome': 'selenium']
     'containerName', //optional configuration in combination with containerMap to define the container where the commands should be executed in
     'containerPortMappings', //map which defines per docker image the port mappings, like containerPortMappings: ['selenium/standalone-chrome': [[name: 'selPort', containerPort: 4444, hostPort: 4444]]]
     'containerShell', // allows to specify the shell to be executed for container with containerName
     'containerWorkspaces', //specify workspace (=home directory of user) per container. If not provided dockerWorkspace will be used. If empty, home directory will not be set.
     'dockerImage',
+    'dockerPullImage',
     'dockerWorkspace',
     'dockerEnvVars',
     'stashContent',
@@ -140,17 +142,17 @@ private void unstashWorkspace(config, prefix) {
 }
 
 private List getContainerList(config) {
-
     result = []
     result.push(containerTemplate(
         name: 'jnlp',
         image: config.jenkinsKubernetes.jnlpAgent
     ))
     config.containerMap.each { imageName, containerName ->
+        def containerPullImage = config.containerPullImageFlags?.get(imageName)
         def templateParameters = [
             name: containerName.toLowerCase(),
             image: imageName,
-            alwaysPullImage: true,
+            alwaysPullImage: containerPullImage != null ? containerPullImage : config.dockerPullImage,
             envVars: getContainerEnvs(config, imageName)
         ]