Merge remote-tracking branch 'upstream/master' into harmonize-docker-arguments

.codeclimate.yml

@@ -18,6 +18,12 @@ plugins:
       strings:
       - TODO
       - FIXME
+  gofmt:
+    enabled: true
+  golint:
+    enabled: true
+  govet:
+    enabled: true
   markdownlint:
     enabled: true
     checks:

.editorconfig

@@ -22,3 +22,6 @@ indent_size = none
 [cfg/id_rsa.enc]
 indent_style = none
 indent_size = none
+[{go.mod,go.sum,*.go}]
+indent_style = tab
+indent_size = 8

.gitignore

@@ -17,3 +17,5 @@ targets/
 documentation/docs-gen
 
 consumer-test/**/workspace
+
+*.code-workspace

.travis.yml

@@ -3,6 +3,8 @@ branches:
   - master
   - /^it\/.*$/
 language: groovy
+jdk:
+  - openjdk8
 sudo: required
 services:
   - docker
@@ -25,7 +27,9 @@ jobs:
         - curl -L --output cc-test-reporter https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
         - chmod +x ./cc-test-reporter
         - ./cc-test-reporter before-build
-      script: mvn package --batch-mode
+      script:
+        - docker build -t piper:latest . 
+        - mvn package --batch-mode
       after_script:
         - JACOCO_SOURCE_PATH="src vars test" ./cc-test-reporter format-coverage target/site/jacoco/jacoco.xml --input-type jacoco
         - ./cc-test-reporter upload-coverage

DEVELOPMENT.md

@@ -0,0 +1,131 @@
+# Development
+
+**Table of contents:**
+
+1. [Getting started](#getting-started)
+1. [Build the project](#build-the-project_)
+1. [Logging](#logging)
+1. [Error handling](#error-handling)
+
+## Getting started
+
+1. [Ramp up your development environment](#ramp-up)
+1. [Get familiar with Go language](#go-basics)
+1. Create [a GitHub account](https://github.com/join)
+1. Setup [GitHub access via SSH](https://help.github.com/articles/connecting-to-github-with-ssh/)
+1. [Create and checkout a repo fork](#checkout-your-fork)
+1. Optional: [Get Jenkins related environment](#jenkins-environment)
+1. Optional: [Get familiar with Jenkins Pipelines as Code](#jenkins-pipelines)
+
+### Ramp up
+
+First you need to set up an appropriate development environment:
+
+Install Go, see [GO Getting Started](https://golang.org/doc/install)
+
+Install an IDE with Go plugins, see for example [Go in Visual Studio Code](https://code.visualstudio.com/docs/languages/go)
+
+### Go basics
+
+In order to get yourself started, there is a lot of useful information out there.
+
+As a first step to take we highly recommend the [Golang documentation](https://golang.org/doc/) especially, [A Tour of Go](https://tour.golang.org/welcome/1)
+
+We have a strong focus on high quality software and contributions without adequate tests will not be accepted.
+There is an excellent resource which teaches Go using a test-driven approach: [Learn Go with Tests](https://github.com/quii/learn-go-with-tests)
+
+### Checkout your fork
+
+The project uses [Go modules](https://blog.golang.org/using-go-modules). Thus please make sure to **NOT** checkout the project into your [`GOPATH`](https://github.com/golang/go/wiki/SettingGOPATH).
+
+To check out this repository:
+
+1. Create your own
+    [fork of this repo](https://help.github.com/articles/fork-a-repo/)
+1. Clone it to your machine, for example like:
+
+```shell
+mkdir -p ${HOME}/projects/jenkins-library
+cd ${HOME}/projects
+git clone git@github.com:${YOUR_GITHUB_USERNAME}/jenkins-library.git
+cd jenkins-library
+git remote add upstream git@github.com:sap/jenkins-library.git
+git remote set-url --push upstream no_push
+```
+
+### Jenkins environment
+
+If you want to contribute also to the Jenkins-specific parts like
+
+* Jenkins library step
+* Jenkins pipeline integration
+
+you need to do the following in addition:
+
+* [Install Groovy](https://groovy-lang.org/install.html)
+* [Install Maven](https://maven.apache.org/install.html)
+* Get a local Jenkins installed: Use for example [cx-server](toDo: add link)
+
+### Jenkins pipelines
+
+The Jenkins related parts depend on
+
+* [Jenkins Pipelines as Code](https://jenkins.io/doc/book/pipeline-as-code/)
+* [Jenkins Shared Libraries](https://jenkins.io/doc/book/pipeline/shared-libraries/)
+
+You should get familiar with these concepts for contributing to the Jenkins-specific parts.
+
+## Build the project
+
+### Build the executable suitable for the CI/CD Linux target environments
+
+Use Docker:
+
+`docker build -t piper:latest .`
+
+You can extract the binary using Docker means to your local filesystem:
+
+```
+docker create --name piper piper:latest
+docker cp piper:/piper .
+docker rm piper
+```
+
+## Generating step framework
+
+The steps are generated based on the yaml files in `resources/metadata/` with the following command
+`go run pkg/generator/step-metadata.go`.
+
+The yaml format is kept pretty close to Tekton's [task format](https://github.com/tektoncd/pipeline/blob/master/docs/tasks.md).
+Where the Tekton format was not sufficient some extenstions have been made.
+
+Examples are:
+
+* matadata - longDescription
+* spec - inputs - secrets
+* spec - containers
+* spec - sidecars
+
+## Logging
+
+to be added
+
+## Error handling
+
+In order to better understand the root cause of errors that occur we wrap errors like
+
+```golang
+    f, err := os.Open(path)
+    if err != nil {
+        return errors.Wrapf(err, "open failed for %v", path)
+    }
+    defer f.Close()
+```
+
+We use [github.com/pkg/errors](https://github.com/pkg/errors) for that.
+
+## Testing
+
+Unit tests are done using basic `golang` means.
+
+Additionally we encourage you to use [github.com/stretchr/testify/assert](https://github.com/stretchr/testify/assert) in order to have slimmer assertions if you like.

Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.13 AS build-env
+COPY . /build
+WORKDIR /build
+
+# execute tests
+RUN go test ./... -cover
+
+## ONLY tests so far, building to be added later
+# execute build
+# RUN go build -o piper
+
+# FROM gcr.io/distroless/base:latest
+# COPY --from=build-env /build/piper /piper
+# ENTRYPOINT ["/piper"]

consumer-test/jenkins.yml

@@ -33,3 +33,9 @@ credentials:
               username: ${NEO_DEPLOY_USERNAME}
               password: ${NEO_DEPLOY_PASSWORD}
               description: "SAP CP NEO Trail account for test deployment"
+          - usernamePassword:
+              scope: GLOBAL
+              id: "cf_deploy"
+              username: ${CX_INFRA_IT_CF_USERNAME}
+              password: ${CX_INFRA_IT_CF_PASSWORD}
+              description: "SAP CP CF Trial account for test deployment"

consumer-test/testCases/scs/cap.yml

@@ -0,0 +1,7 @@
+# Test case configuration
+referenceAppRepo:
+  url: "https://github.com/piper-validation/mta-sample-app.git"
+  branch: "piper-test-cap"
+deployCredentialEnv:
+  username: "CX_INFRA_IT_CF_USERNAME"
+  password: "CX_INFRA_IT_CF_PASSWORD"

documentation/bin/createDocu.groovy

@@ -364,7 +364,7 @@ class Helper {
                         def param = retrieveParameterName(line)
 
                         if(!param) {
-                            throw new RuntimeException('Cannot retrieve parameter for a comment')
+                            throw new RuntimeException("Cannot retrieve parameter for a comment. Affected line was: '${line}'")
                         }
 
                         def _docu = [], _value = [], _mandatory = [], _parentObject = []
@@ -489,7 +489,7 @@ class Helper {
         def params = [] as Set
         f.eachLine  {
             line ->
-                if (line ==~ /.*withMandatoryProperty.*/) {
+                if (line ==~ /.*withMandatoryProperty\(.*/) {
                     def param = (line =~ /.*withMandatoryProperty\('(.*)'/)[0][1]
                     params << param
                 }
@@ -666,13 +666,15 @@ Map stages = Helper.resolveDocuRelevantStages(gse, stepsDir)
 boolean exceptionCaught = false
 
 def stepDescriptors = [:]
-DefaultValueCache.prepare(Helper.getDummyScript('noop'),  customDefaults)
+DefaultValueCache.prepare(Helper.getDummyScript('noop'), [customDefaults: customDefaults])
 for (step in steps) {
     try {
         stepDescriptors."${step}" = handleStep(step, gse)
     } catch(Exception e) {
         exceptionCaught = true
-        System.err << "${e.getClass().getName()} caught while handling step '${step}': ${e.getMessage()}.\n"
+        def writer = new StringWriter()
+        e.printStackTrace(new PrintWriter(writer))
+        System.err << "${e.getClass().getName()} caught while handling step '${step}': ${e.getMessage()}.\n${writer.toString()}\n"
     }
 }
 

documentation/docs/configuration.md

@@ -1,6 +1,9 @@
 # Configuration
 
-Configuration is done via a yml-file, located at `.pipeline/config.yml` in the **master branch** of your source code repository.
+Configure your project through a yml-file, which is located at `.pipeline/config.yml` in the **master branch** of your source code repository.
+
+!!! note "Cloud SDK Pipeline"
+    Cloud SDK Pipelines are configured in a file called `pipeline_config.yml`. See [SAP Cloud SDK Pipeline Configuration Docs](https://github.com/SAP/cloud-s4-sdk-pipeline/blob/master/configuration.md).
 
 Your configuration inherits from the default configuration located at [https://github.com/SAP/jenkins-library/blob/master/resources/default_pipeline_environment.yml](https://github.com/SAP/jenkins-library/blob/master/resources/default_pipeline_environment.yml).
 

documentation/docs/extensibility.md

@@ -4,10 +4,13 @@ There are several possibilities for extensibility besides the **[very powerful c
 
 ## 1. Stage Exits
 
-  You have to create a file like `<StageName>.groovy` for example `Acceptance.groovy` and store it in folder `.pipeline/extensions/` in your source code repository.
+  You have to create a file like `<StageName>.groovy` (for example, `Acceptance.groovy`) and store it in folder `.pipeline/extensions/` in your source code repository.
 
-  The pipeline template will check if such a file exists and executes it if present.
-  A parameter is passed to the extension containing following keys:
+!!! note "Cloud SDK Pipeline"
+    If you use the Cloud SDK Pipeline, the folder is named `pipeline/extensions/` (without the dot). For more information, please refer to [the Cloud SDK Pipeline documentation](https://github.com/SAP/cloud-s4-sdk-pipeline/blob/master/doc/pipeline/extensibility.md).
+
+  The pipeline template checks if such a file exists and executes it, if present.
+  A parameter that contains the following keys is passed to the extension:
 
   * `script`: defines the global script environment of the Jenkinsfile run. This makes sure that the correct configuration environment can be passed to project "Piper" steps and also allows access to for example the `commonPipelineEnvironment`.
   * `originalStage`: this will allow you to execute the "original" stage at any place in your script. If omitting a call to `originalStage()` only your code will be executed instead.

documentation/docs/guidedtour.md

@@ -12,29 +12,46 @@ The stated instructions assume the use of this application.
 
 * You have installed a Linux system with at least 4 GB memory. **Note:** We have tested our samples on Ubuntu 16.04. On Microsoft Windows, you might face some issues.
 * You have installed the newest version of Docker. See [Docker Community Edition](https://docs.docker.com/install/). **Note:** we have tested on Docker 18.09.6.
-* You have installed Jenkins 2.60.3 or higher. **Recommendation:** We recommend to use the `cx-server` toolkit. See **(Optional) Install the `cx-server` Toolkit for Jenkins**. **Note:** If you use your **own Jenkins installation** you need to care for "Piper" specific configuration. Follow [my own Jenkins installation][guidedtour-my-own-jenkins].
 * Your system has access to [GitHub.com][github].
 
-## (Optional) Install the `cx-server` Toolkit for Jenkins
+## **Recommended:** Install the Cx Server Life-cycle Management for Jenkins
 
-`cx-server`is a lifecycle management toolkit that provides Docker images with a preconfigured Jenkins and a Nexus-based cache to facilitate the configuration and usage of Jenkins.
+Cx Server is a life-cycle management tool to bootstrap a pre-configured Jenkins instance within minutes.
+All required plugins and shared libraries are included automatically.
+It is based on Docker images provided by project "Piper".
 
-To use the toolkit, get the `cx-server` script and its configuration file `server.cfg` by using the following command:
+To get started, initialize Cx Server by using this `docker run` command:
 
 ```sh
 docker run -it --rm -u $(id -u):$(id -g) -v "${PWD}":/cx-server/mount/ ppiper/cx-server-companion:latest init-cx-server
 ```
 
-When the files are downloaded into the current directory, launch the Jenkins server by using the following command:
+This creates a few files in your current working directory.
+The shell script `cx-server` and the configuration file `server.cfg` are of special interest.
+
+Now, you can start the Jenkins server by using the following command:
 
 ```sh
+chmod +x ./cx-server
 ./cx-server start
 ```
 
-For more information on the Jenkins lifecycle management and how to customize your Jenkins, have a look at the [Operations Guide for Cx Server][devops-docker-images-cxs-guide].
+For more information on the Cx Server and how to customize your Jenkins, have a look at the [Operations Guide for Cx Server][devops-docker-images-cxs-guide].
+
+### On your own: Custom Jenkins Setup
+
+If you use your own Jenkins installation, you need to care for the configuration that is specific to project "Piper".
+This option should only be considered if you know why you need it, otherwise using the Cx Server life-cycle management makes your life much easier.
+If you choose to go this path, follow [my own Jenkins installation][guidedtour-my-own-jenkins] for some hints.
+
+**Note:** This option is not supported for SAP Cloud SDK projects.
 
 ## (Optional) Sample Application
 
+!!! info "Choosing the best sample application"
+    Depending on the type of project you're interested in, different sample applications might be interesting.
+    For SAP Cloud SDK, please have a look at the [Address Manager](https://github.com/sap/cloud-s4-sdk-book) example application.
+
 Copy the sources of the application into your own Git repository. While we will ask you to fork the application's repository into a **GitHub** space, you can use any version control system based on Git like **GitLab** or **plain git**. **Note:** A `public` GitHub repository is visible to the public. The configuration files may contain data you don't want to expose, so use a `private` repository.
 
 1. Create an organization on GitHub, if you haven't any yet. See [Creating a new organization][github-create-org].
@@ -189,7 +206,7 @@ Please also consult the blog post on setting up [Continuous Delivery for S/4HANA
 [sap-blog-s4-sdk-first-steps]:       https://blogs.sap.com/2017/05/10/first-steps-with-sap-s4hana-cloud-sdk/
 [sap-blog-ci-cd]:                    https://blogs.sap.com/2017/09/20/continuous-integration-and-delivery/
 
-[devops-docker-images-cxs-guide]:    https://github.com/SAP/devops-docker-images/blob/master/docs/operations/cx-server-operations-guide.md
+[devops-docker-images-cxs-guide]:    https://github.com/SAP/devops-docker-cx-server/blob/master/docs/operations/cx-server-operations-guide.md
 
 [cloud-cf-helloworld-nodejs]:        https://github.com/SAP/cloud-cf-helloworld-nodejs
 [github]:                            https://github.com

documentation/docs/index.md

@@ -1,32 +1,45 @@
 # Project "Piper" User Documentation
 
-An efficient software development process is vital for success in building
-business applications on SAP Cloud Platform or SAP on-premise platforms. SAP
-addresses this need for efficiency with project "Piper". The goal of project
-"Piper" is to substantially ease setting up continuous delivery processes for
-the most important SAP technologies by means of Jenkins pipelines.
+Continuous delivery is a method to develop software with short feedback cycles.
+It is applicable to projects both for SAP Cloud Platform and SAP on-premise platforms.
+SAP implements tooling for continuous delivery in project "Piper".
+The goal of project "Piper" is to substantially ease setting up continuous delivery in your project using SAP technologies.
 
 ## What you get
 
-Project "Piper" consists of two parts:
+To get you started quickly, project "Piper" offers you the following artifacts:
 
-* [A shared library][piper-library] containing steps and utilities that are
-  required by Jenkins pipelines.
-* A set of [Docker images][devops-docker-images] used in the piper library to implement best practices.
+* A set of ready-made Continuous Delivery pipelines for direct use in your project
+    * [General Purpose Pipeline](stages/introduction/)
+    * [SAP Cloud SDK Pipeline][cloud-sdk-pipeline]
+* [A shared library][piper-library] that contains reusable step implementations, which enable you to customize our preconfigured pipelines, or to even build your own customized ones
+* A set of [Docker images][devops-docker-images] to setup a CI/CD environment in minutes using sophisticated life-cycle management
 
-The shared library contains all the necessary steps to run our best practice
-[Jenkins pipelines][piper-library-pages] described in the Scenarios section or
-to run a [pipeline as step][piper-library-scenario].
+To find out which offering is right for you, we recommend to look at the ready-made pipelines first.
+In many cases, they should satisfy your requirements, and if this is the case, you don't need to build your own pipeline.
 
-The best practice pipelines are based on the general concepts of [Jenkins 2.0
-Pipelines as Code][jenkins-doc-pipelines].  With that you have the power of the
-Jenkins community at hand to optimize your pipelines.
+### The best-practice way: Ready-made pipelines
+
+**Are you building a standalone SAP Cloud Platform application?<br>**
+Then continue reading about our [general purpose pipeline](stages/introduction/), which supports various technologies and programming languages.
+
+**Are you building an application with the SAP Cloud SDK and/or SAP Cloud Application Programming Model?<br>**
+Then we can offer you a [pipeline specifically tailored to SAP Cloud SDK and SAP Cloud Application Programming Model applications][cloud-sdk-pipeline]
+
+### The do-it-yourself way: Build with Library
+
+The shared library contains building blocks for your own pipeline, following our best practice Jenkins pipelines described in the Scenarios section.
+
+The best practice pipelines are based on the general concepts of [Pipelines as Code, as introduced in Jenkins 2][jenkins-doc-pipelines].
+With that you have the power of the Jenkins community at hand to optimize your pipelines.
 
 You can run the best practice Jenkins pipelines out of the box, take them as a
 starting point for project-specific adaptations or implement your own pipelines
 from scratch using the shared library.
 
-## Extensibility
+For an example, you might want to check out our ["Build and Deploy SAPUI5 or SAP Fiori Applications on SAP Cloud Platform with Jenkins" scenario][piper-library-scenario].
+
+#### Extensibility
 
 If you consider adding additional capabilities to your `Jenkinsfile`, consult
 the [Jenkins Pipeline Steps Reference][jenkins-doc-steps]. There, you get an
@@ -41,7 +54,7 @@ Custom library steps can be added using a custom library according to the
 groovy coding to the `Jenkinsfile`. Your custom library can coexist next to the
 provided pipeline library.
 
-## API
+#### API
 
 All steps (`vars` and `resources` directory) are intended to be used by Pipelines and are considered API.
 All the classes / groovy-scripts contained in the `src` folder are by default not part of
@@ -49,14 +62,15 @@ the API and are subjected to change without prior notice. Types and methods anno
 `@API` are considered to be API, used e.g. from other shared libraries. Changes to those
 methods/types needs to be announced, discussed and agreed.
 
+
 [github]: https://github.com
 [piper-library]: https://github.com/SAP/jenkins-library
+[cloud-sdk-pipeline]: pipelines/cloud-sdk/introduction/
 [devops-docker-images]: https://github.com/SAP/devops-docker-images
 [devops-docker-images-issues]:       https://github.com/SAP/devops-docker-images/issues
 [devops-docker-images-cxs-guide]:     https://github.com/SAP/devops-docker-images/blob/master/docs/operations/cx-server-operations-guide.md
-[piper-library-scenario]: https://sap.github.io/jenkins-library/scenarios/ui5-sap-cp/Readme/
-[piper-library-pages]: https://sap.github.io/jenkins-library
-[piper-library-pages-plugins]: https://sap.github.io/jenkins-library/jenkins/requiredPlugins
+[piper-library-scenario]: scenarios/ui5-sap-cp/Readme/
+[piper-library-pages-plugins]: requiredPlugins
 [piper-library-issues]: https://github.com/SAP/jenkins-library/issues
 [piper-library-license]: ./LICENSE
 [piper-library-contribution]: .github/CONTRIBUTING.md

documentation/docs/pipelines/cloud-sdk/introduction.md

@@ -0,0 +1,41 @@
+# SAP Cloud SDK Pipeline
+
+<img src="https://help.sap.com/doc/6c02295dfa8f47cf9c08a19f2e172901/1.0/en-US/logo-for-cd.svg" alt="SAP Cloud SDK for Continuous Delivery Logo" height="122.92" width="226.773" align="right"/></a>
+
+If you are building an application with [SAP Cloud SDK](https://community.sap.com/topics/cloud-sdk), the [SAP Cloud SDK pipeline](https://github.com/SAP/cloud-s4-sdk-pipeline) helps you to quickly build and deliver your app in high quality.
+Thanks to highly streamlined components, setting up and delivering your first project will just take minutes.
+
+## Qualities and Pipeline Features
+
+The SAP Cloud SDK pipeline is based on project "piper" and offers unique features for assuring that your SAP Cloud SDK based application fulfills highest quality standards.
+In conjunction with the SAP Cloud SDK libraries, the pipeline helps you to implement and automatically assure application qualities, for example:
+
+* Functional correctness via:
+    * Backend and frontend unit tests
+    * Backend and frontend integration tests
+    * User acceptance testing via headless browser end-to-end tests
+* Non-functional qualities via:
+    * Dynamic resilience checks
+    * Performance tests based on *Gatling* or *JMeter*
+    * Code Security scans based on *Checkmarx* and *Fortify*
+    * Dependency vulnerability scans based on *Whitesource*
+    * IP compliance scan based on *Whitesource*
+    * Zero-downtime deployment
+    * Proper logging of application errors
+
+![Screenshot of SAP Cloud SDK Pipeline](../../images/cloud-sdk-pipeline.png)
+
+## Supported Project Types
+
+The pipeline supports the following types of projects:
+
+* Java projects based on the [SAP Cloud SDK Archetypes](https://mvnrepository.com/artifact/com.sap.cloud.sdk.archetypes).
+* JavaScript projects based on the [SAP Cloud SDK JavaScript Scaffolding](https://github.com/SAP/cloud-s4-sdk-examples/tree/scaffolding-js).
+* TypeScript projects based on the [SAP Cloud SDK TypeScript Scaffolding](https://github.com/SAP/cloud-s4-sdk-examples/tree/scaffolding-ts).
+* SAP Cloud Application Programming Model (CAP) projects based on the _SAP Cloud Platform Business Application_ WebIDE Template.
+
+You can find more details about the supported project types and build tools in the [project documentation](https://github.com/SAP/cloud-s4-sdk-pipeline/blob/master/doc/pipeline/build-tools.md).
+
+## Legal Notes
+
+Note: This license of this repository does not apply to the SAP Cloud SDK for Continuous Delivery Logo referenced in this page

documentation/docs/scenarios/CAP_Scenario.md

@@ -1,9 +1,49 @@
-# Build and Deploy Applications with Jenkins and the SAP Cloud Application Programming Model
+# Build and Deploy SAP Cloud Application Programming Model Applications
 
-Set up a basic continuous delivery process for developing applications according to the SAP Cloud Application Programming Model.
+In this scenario, we will setup a CI/CD Pipeline for a SAP Cloud Application Programming Model (CAP) project, which is based on the _SAP Cloud Platform Business Application_ WebIDE Template.
 
 ## Prerequisites
 
+* You have an account on SAP Cloud Platform in the Cloud Foundry environment. See [Accounts](https://help.sap.com/viewer/65de2977205c403bbc107264b8eccf4b/Cloud/en-US/8ed4a705efa0431b910056c0acdbf377.html).
+* You have setup a suitable Jenkins instance as described in [Guided Tour](../guidedtour.md)
+
+## Context
+
+The Application Programming Model for SAP Cloud Platform is an end-to-end best practice guide for developing applications on SAP Cloud Platform and provides a supportive set of APIs, languages, and libraries.
+For more information about the SAP Cloud Application Programming Model, see [Working with the SAP Cloud Application Programming Model](https://help.sap.com/viewer/65de2977205c403bbc107264b8eccf4b/Cloud/en-US/00823f91779d4d42aa29a498e0535cdf.html).
+
+## Getting started
+
+To get started, generate a project in SAP Web IDE based on the _SAP Cloud Platform Business Application_ template.
+Make sure to check the Include support for continuous delivery pipeline of SAP Cloud SDK checkbox, as in this screenshot:
+
+![WebIDE project wizard](../images/webide-pipeline-template.png)
+
+This will generate a project which already includes a `Jenkinsfile`, and a `pipeline_config.yml` file.
+
+In case you already created your project without this option, you'll need to copy and paste two files into the root directory of your project, and commit them to your git repository:
+
+* [`Jenkinsfile`](https://github.com/SAP/cloud-s4-sdk-pipeline/blob/master/archetype-resources/Jenkinsfile)
+* [`pipeline_config.yml`](https://github.com/SAP/cloud-s4-sdk-pipeline/blob/master/archetype-resources/cf-pipeline_config.yml)
+    * Note: The file must be named `pipeline_config.yml`, despite the different name of the file template
+
+!!! note "Using the right project structure"
+    This only applies to projects created based on the _SAP Cloud Platform Business Application_ template after September 6th 2019. They must comply with the structure which is described [here](https://github.com/SAP/cloud-s4-sdk-pipeline/blob/master/doc/pipeline/build-tools.md#sap-cloud-application-programming-model--mta).
+
+If your project uses SAP HANA containers (HDI), you'll need to configure `createHdiContainer` and `cloudFoundry` in the `backendIntegrationTests` stage in your `pipeline_config.yml` file as documented [here](https://github.com/SAP/cloud-s4-sdk-pipeline/blob/master/configuration.md#backendintegrationtests)
+
+Now, you'll need to push the code to a git repository.
+This is required because the pipeline gets your code via git.
+This might be GitHub, or any other cloud or on-premise git solution you have in your company.
+
+Be sure to configure the [`productionDeployment `](https://github.com/SAP/cloud-s4-sdk-pipeline/blob/master/configuration.md#productiondeployment) stage so your changes are deployed to SAP Cloud Platform automatically.
+
+## Legacy documentation
+
+If your project is not based on the _SAP Cloud Platform Business Application_ WebIDE template, you could either migrate your code to comply with the structure which is described [here](https://github.com/SAP/cloud-s4-sdk-pipeline/blob/master/doc/pipeline/build-tools.md#sap-cloud-application-programming-model--mta), or you can use a self built pipeline, as described in this section.
+
+### Prerequisites
+
 * You have an account on SAP Cloud Platform in the Cloud Foundry environment. See [Accounts](https://help.sap.com/viewer/65de2977205c403bbc107264b8eccf4b/Cloud/en-US/8ed4a705efa0431b910056c0acdbf377.html).
 * You have downloaded and installed the Cloud Foundry command line interface (CLI). See [Download and Install the Cloud Foundry Command Line Interface](https://help.sap.com/viewer/65de2977205c403bbc107264b8eccf4b/Cloud/en-US/afc3f643ec6942a283daad6cdf1b4936.html).
 * You have installed the multi-target application plug-in for the Cloud Foundry command line interface. See [Install the Multi-Target Application Plug-in in the Cloud Foundry Environment](https://help.sap.com/viewer/65de2977205c403bbc107264b8eccf4b/Cloud/en-US/27f3af39c2584d4ea8c15ba8c282fd75.html).
@@ -13,15 +53,15 @@ Set up a basic continuous delivery process for developing applications according
 * You have installed the Multi-Target Application (MTA) Archive Builder 1.0.6 or newer. See [SAP Development Tools](https://tools.hana.ondemand.com/#cloud).
 * You have installed Node.js including node and npm. See [Node.js](https://nodejs.org/en/download/).
 
-## Context
+### Context
 
 The Application Programming Model for SAP Cloud Platform is an end-to-end best practice guide for developing applications on SAP Cloud Platform and provides a supportive set of APIs, languages, and libraries. For more information about the SAP Cloud Application Programming Model, see [Working with the SAP Cloud Application Programming Model](https://help.sap.com/viewer/65de2977205c403bbc107264b8eccf4b/Cloud/en-US/00823f91779d4d42aa29a498e0535cdf.html).
 
 In this scenario, we want to show how to implement a basic continuous delivery process for developing applications according to this programming model with the help of project "Piper" on Jenkins. This basic scenario can be adapted and enriched according to your specific needs.
 
-## Example
+### Example
 
-### Jenkinsfile
+#### Jenkinsfile
 
 ```groovy
 @Library('piper-library-os') _
@@ -43,7 +83,7 @@ node(){
 }
 ```
 
-### Configuration (`.pipeline/config.yml`)
+#### Configuration (`.pipeline/config.yml`)
 
 ```yaml
 steps:
@@ -57,9 +97,9 @@ steps:
       space: '<CF Space>'
 ```
 
-### Parameters
+#### Parameters
 
 For the detailed description of the relevant parameters, see:
 
-* [mtaBuild](https://sap.github.io/jenkins-library/steps/mtaBuild/)
-* [cloudFoundryDeploy](https://sap.github.io/jenkins-library/steps/cloudFoundryDeploy/)
+* [mtaBuild](../../../steps/mtaBuild/)
+* [cloudFoundryDeploy](../../../steps/cloudFoundryDeploy/)

documentation/docs/scenarios/TMS_Extension.md

@@ -0,0 +1,77 @@
+# Integrate SAP Cloud Platform Transport Management Into Your CI/CD Pipeline
+
+Extend your CI/CD pipeline with SAP Cloud Platform Transport Management to add an enterprise-ready change and release management process and enable the transport of cloud-based applications on SAP Cloud Platform between several stages.
+
+## Context
+
+This procedure explains how to upload a [multitartget application](https://www.sap.com/documents/2016/06/e2f618e4-757c-0010-82c7-eda71af511fa.html) from a CI/CD pipeline to SAP Cloud Platform Transport Management and then import it into its target environment.
+
+SAP Cloud Platform Transport Management allows you to manage the transport of development artifacts and application-specific content between different SAP Cloud Platform accounts. It adds transparency to the audit trail of changes so that you get information about who performed which changes in your production accounts and when they did it. At the same time, the Transport Management service enables a separation of concerns: For example, a developer of an application or SAP Cloud Platform content artifacts can trigger the propagation of changes, while the resulting transport is handled by a central operations team. For more information, see [SAP Cloud Platform Transport Management](https://help.sap.com/viewer/product/TRANSPORT_MANAGEMENT_SERVICE/Cloud/en-US).
+
+The following graphic provides an overview about the interplay between continuous integration and Transport Management:
+
+![Interplay of CI and Transport Management](../images/Interplay_TMS.png "Interplay of CI and Transport Management")
+
+## Prerequisites
+
+* You have an existing CI pipeline, which you want to extend with SAP Cloud Platform Transport Management.
+* You have an MTA project and the folder structure of its sources corresponds to the standard MTA structure. For more information, see [The Multitarget Application Model](https://www.sap.com/documents/2016/06/e2f618e4-757c-0010-82c7-eda71af511fa.html).
+* You have access to SAP Cloud Platform Transport Management. See [Provide Access to SAP Cloud Platform Transport Management](https://help.sap.com/viewer/7f7160ec0d8546c6b3eab72fb5ad6fd8/Cloud/en-US/13894bed9e2d4b25aa34d03d002707f9.html).
+* You have set up SAP Cloud Platform Transport Management and created a service key. See [Set Up the Environment to Transport Content Archives directly in an Application](https://help.sap.com/viewer/7f7160ec0d8546c6b3eab72fb5ad6fd8/Cloud/en-US/8d9490792ed14f1bbf8a6ac08a6bca64.html).
+* You have configured your Transport Management landscape. See [Configuring the Landscape](https://help.sap.com/viewer/7f7160ec0d8546c6b3eab72fb5ad6fd8/Cloud/en-US/3e7b04236d804a4eb80e42c6360209f1.html).
+
+## Procedure
+
+You can use this scenario to extend any CI process that meets the prerequisites, for example, the one described in [Build and Deploy SAPUI5 or SAP Fiori Applications on SAP Cloud Platform with Jenkins](https://sap.github.io/jenkins-library/scenarios/ui5-sap-cp/Readme/).
+
+The following graphic shows an example of the detailed procedure when combining continuous integration and SAP Cloud Platform Transport Management:
+
+![Detailed Procedure When Combining CI and SAP Cloud Platform Transport Management](../images/Detailed_Process_TMS.png "Detailed Procedure When Combining CI and SAP Cloud Platform Transport Management")
+
+The process flow contains the following steps:
+
+1. The CI server builds a multitarget application (MTA) archive.
+1. The MTA is uploaded into the import queue of the target node, which is specified in the CI pipeline (in this example, PRE-PROD).
+1. The release manager manually triggers or schedules the import, which results in the physical deployment of the MTA archive into the corresponding subaccount (in this example, PRE-PROD).
+1. As soon as the import is executed, a transport is triggered along the defined transport route so that the MTA archive reaches the import queue of the next node (in this example, PROD).
+1. There, the physical import into the corresponding subaccount can be either triggered manually by the release manager or automatically by using the scheduling mechanisms of SAP Cloud Platform Transport Management.
+
+## Example
+
+### Jenkinsfile
+
+If you use the pipeline of the following code snippet, you only have to configure it in the .pipeline/config.yml.
+
+Following the convention for pipeline definitions, use a Jenkinsfile, which resides in the root directory of your development sources.
+
+```groovy
+@Library('piper-lib-os') _
+
+piperPipeline script:this
+```
+
+### Configuration (`.pipeline/config.yml`)
+
+This is a basic configuration example, which is also located in the sources of the project.
+
+```yaml
+steps:
+  tmsUpload:
+    credentialsId: tms-secret-key
+    nodeName: tms_target_node
+    mtaPath: com.piper.example.tms.mtar
+    customDescription: Custom-Transport-Description
+```
+
+#### Configration for the Upload to Transport Management
+
+| Parameter          | Description |
+| -------------------|-------------|
+| `credentialsId` |Credentials that are used for the file and node uploads to the Transport Management Service.|
+| `nodeName`|Defines the name of the node to which the *.mtar file is uploaded.|
+| `mtaPath`|Defines the path to *.mtar for the upload to the Transport Management Service.|
+| `customDescription`| Can be used as description of a transport request. Overwrites the default (Default: Corresponding Git Commit-ID).|
+
+### Parameters
+
+For a detailed description of the relevant parameters, see [tmsUpload](../../../steps/tmsUpload/).

documentation/docs/scenarios/changeManagement.md

@@ -8,8 +8,8 @@ Set up an agile development process with Jenkins CI, which automatically feeds c
 * You have installed Jenkins 2.60.3 or higher.
 * You have set up Project “Piper”. See [README](https://github.com/SAP/jenkins-library/blob/master/README.md).
 * You have installed SAP Solution Manager 7.2 SP6. See [README](https://github.com/SAP/devops-cm-client/blob/master/README.md).
-* You have installed the Multi-Target Application (MTA) Archive Builder 1.0.6 or newer. See [SAP Development Tools](https://tools.hana.ondemand.com/#cloud).
-* You have installed Node.js including node and npm. See [Node.js](https://nodejs.org/en/download/).
+* You have installed the Multi-Target Application (MTA) Archive Builder 1.0.6 or newer. See [SAP Development Tools](https://tools.hana.ondemand.com/#cloud). **Note:** This is only required if you don't use a Docker-based environment.
+* You have installed Node.js including node and npm. See [Node.js](https://nodejs.org/en/download/).  **Note:** This is only required if you don't use a Docker-based environment.
 
 ## Context
 
@@ -22,7 +22,7 @@ In this scenario, we want to show how an agile development process with Jenkins
 
 The basic workflow is as follows:
 
-1. The pipeline scans the Git commit messages for a line like `ChangeDocument : <changeDocumentId>`, and validates that the change is in the correct status `in development`. For more information, see [checkChangeInDevelopment](https://sap.github.io/jenkins-library/steps/checkChangeInDevelopment/). An example for the commit message looks as follows:
+1. The pipeline scans the Git commit messages for a line like `ChangeDocument : <changeDocumentId>`, and validates that the change is in the correct status `in development`. For more information, see [checkChangeInDevelopment](../../steps/checkChangeInDevelopment/). An example for the commit message looks as follows:
 
     ```
     Fix terminology in documentation
@@ -33,7 +33,7 @@ The basic workflow is as follows:
 
     **Note:** The blank line between message header and message description is mandatory.
 
-1. To communicate with SAP Solution Manager, the pipeline uses credentials that must be stored on Jenkins using the credential ID `CM`. For more information, see [checkChangeInDevelopment](https://sap.github.io/jenkins-library/steps/checkChangeInDevelopment/).
+1. To communicate with SAP Solution Manager, the pipeline uses credentials that must be stored on Jenkins using the credential ID `CM`. For more information, see [checkChangeInDevelopment](../../steps/checkChangeInDevelopment/).
 1. The required transport request is created on the fly. **Note:** The change document can contain various components (for example, UI and backend components).
 1. The changes of your development team trigger the Jenkins pipeline. It builds and validates the changes and attaches them to the respective transport request.
 1. As soon as the development process is completed, the change document in SAP Solution Manager can be set to status `to be tested` and all components can be transported to the test system.
@@ -91,8 +91,8 @@ steps:
 
 For the detailed description of the relevant parameters, see:
 
-* [checkChangeInDevelopment](https://sap.github.io/jenkins-library/steps/checkChangeInDevelopment/)
-* [mtaBuild](https://sap.github.io/jenkins-library/steps/mtaBuild/)
-* [transportRequestCreate](https://sap.github.io/jenkins-library/steps/transportRequestCreate/)
-* [transportRequestUploadFile](https://sap.github.io/jenkins-library/steps/transportRequestUploadFile/)
-* [transportRequestRelease](https://sap.github.io/jenkins-library/steps/transportRequestRelease/)
+* [checkChangeInDevelopment](../../steps/checkChangeInDevelopment/)
+* [mtaBuild](../../steps/mtaBuild/)
+* [transportRequestCreate](../../steps/transportRequestCreate/)
+* [transportRequestUploadFile](../../steps/transportRequestUploadFile/)
+* [transportRequestRelease](../../steps/transportRequestRelease/)

documentation/docs/scenarios/ui5-sap-cp/Readme.md

@@ -28,7 +28,7 @@ On the project level, provide and adjust the following template:
 
 This scenario combines various different steps to create a complete pipeline.
 
-In this scenario, we want to show how to build an application based on SAPUI5 or SAP Fiori by using the multi-target application (MTA) concept and how to deploy the build result into an SAP Cloud Platform account in the Neo environment. This document comprises the [mtaBuild](https://sap.github.io/jenkins-library/steps/mtaBuild/) and the [neoDeploy](https://sap.github.io/jenkins-library/steps/neoDeploy/) steps.
+In this scenario, we want to show how to build an application based on SAPUI5 or SAP Fiori by using the multi-target application (MTA) concept and how to deploy the build result into an SAP Cloud Platform account in the Neo environment. This document comprises the [mtaBuild](../../../steps/mtaBuild/) and the [neoDeploy](../../../steps/neoDeploy/) steps.
 
 ![This pipeline in Jenkins Blue Ocean](images/pipeline.jpg)
 ###### Screenshot: Build and Deploy Process in Jenkins
@@ -82,5 +82,5 @@ steps:
 
 For the detailed description of the relevant parameters, see:
 
-* [mtaBuild](https://sap.github.io/jenkins-library/steps/mtaBuild/)
-* [neoDeploy](https://sap.github.io/jenkins-library/steps/neoDeploy/)
+* [mtaBuild](../../../steps/mtaBuild/)
+* [neoDeploy](../../../steps/neoDeploy/)

documentation/docs/scenarios/xsa-deploy/Readme.md

@@ -0,0 +1,101 @@
+# Build and Deploy SAP Fiori Applications on SAP HANA XS Advanced
+
+Build an application based on SAPUI5 or SAP Fiori with Jenkins and deploy the build result into an SAP Cloud Platform account in the Neo environment.
+
+## Prerequisites
+
+* [Docker environment](https://docs.docker.com/get-started/)
+* All artifacts refereneced during the build are available either on Service Market Place or via public repositories
+* You have set up Project “Piper”. See [guided tour](https://sap.github.io/jenkins-library/guidedtour/).
+* Docker image for xs deployment available. Due to legal reasons there is no pre-build docker image. How to create the docker image is explained [here](https://github.com/SAP/devops-docker-images/tree/master/xs-cli).
+
+### Project Prerequisites
+
+This scenario requires additional files in your project and in the execution environment on your Jenkins instance.
+For details see: [XSA developer quick start guide](https://help.sap.com/viewer/400066065a1b46cf91df0ab436404ddc/2.0.04/en-US/7f681c32c2a34735ad85e4ab403f8c26.html).
+
+## Context
+
+This scenario combines various different steps to create a complete pipeline.
+
+In this scenario, we want to show how to build a Multitarget Application (MTA) and deploy the build result into an on-prem SAP HANA XS advances system. This document comprises the [mtaBuild](https://sap.github.io/jenkins-library/steps/mtaBuild/) and the [xsDeploy](https://sap.github.io/jenkins-library/steps/xsDeploy/) steps.
+
+![This pipeline in Jenkins Blue Ocean](images/pipeline.jpg)
+###### Screenshot: Build and Deploy Process in Jenkins
+
+## Example
+
+### Jenkinsfile
+
+Following the convention for pipeline definitions, use a `Jenkinsfile`, which resides in the root directory of your development sources.
+
+```groovy
+@Library('piper-lib-os') _
+
+pipeline {
+
+    agent any
+
+    stages {
+        stage("prepare") {
+            steps {
+                deleteDir()
+                checkout scm
+                setupCommonPipelineEnvironment script: this
+            }
+        }
+        stage('build') {
+            steps {
+                mtaBuild script: this
+            }
+        }
+        stage('deploy') {
+            steps {
+                xsDeploy script: this
+            }
+        }
+    }
+}
+```
+
+### Configuration (`.pipeline/config.yml`)
+
+This is a basic configuration example, which is also located in the sources of the project.
+
+```yaml
+steps:
+  mtaBuild:
+    buildTarget: 'XSA'
+  xsDeploy:
+    apiUrl: '<API_URL>' # e.g. 'https://example.org:30030'
+    # credentialsId: 'XS' omitted, 'XS' is the default
+    docker:
+      dockerImage: '<ID_OF_THE_DOCKER_IMAGE' # for legal reasons no docker image is provided.
+      # dockerPullImage: true # default: 'false'. Needs to be set to 'true' in case the image is served from a docker registry
+    loginOpts: '' # during setup for non-productive builds we might set here. '--skip-ssl-validation'
+    org: '<ORG_NAME>'
+    space: '<SPACE>'
+
+```
+
+#### Configuration for the MTA Build
+
+| Parameter        | Description    |
+| -----------------|----------------|
+| `buildTarget`    | The target platform to which the mtar can be deployed. In this case, the target platform is  `XSA`. |
+
+#### Configuration for the Deployment to XSA
+
+| Parameter          | Description |
+| -------------------|-------------|
+| `credentialsId` | The Jenkins credentials that contain user and password required for the deployment on SAP Cloud Platform.|
+| `mode`          | DeployMode. See [stepDocu](../../../steps/xsDeploy) for more details. |
+| `org`           | The org. See [stepDocu](../../../steps/xsDeploy) for more details. |
+| `space`         | The space. See [stepDocu](../../../steps/xsDeploy) for more details. |
+
+### Parameters
+
+For the detailed description of the relevant parameters, see:
+
+* [mtaBuild](https://sap.github.io/jenkins-library/steps/mtaBuild/)
+* [xsDeploy](https://sap.github.io/jenkins-library/steps/xsDeploy/)

documentation/docs/steps/cfManifestSubstituteVariables.md

@@ -0,0 +1,73 @@
+# ${docGenStepName}
+
+## ${docGenDescription}
+
+## ${docGenParameters}
+
+## ${docGenConfiguration}
+
+## ${docJenkinsPluginDependencies}
+
+## Side effects
+
+Unless configured otherwise, this step will *replace* the input `manifest.yml` with a version that has all variable references replaced. This alters the source tree in your Jenkins workspace.
+If you prefer to generate a separate output file, use the step's `outputManifestFile` parameter. Keep in mind, however, that your Cloud Foundry deployment step should then also reference this output file - otherwise CF deployment will fail with unresolved variable reference errors.
+
+## Exceptions
+
+* `org.yaml.snakeyaml.scanner.ScannerException` - in case any of the loaded input files contains malformed Yaml and cannot be parsed.
+
+* `hudson.AbortException` - in case of internal errors and when not all variables could be replaced due to missing replacement values.
+
+## Example
+
+Usage of pipeline step:
+
+```groovy
+cfManifestSubstituteVariables (
+  script: this,
+  manifestFile: "path/to/manifest.yml",                      //optional, default: manifest.yml
+  manifestVariablesFiles: ["path/to/manifest-variables.yml"] //optional, default: ['manifest-variables.yml']
+  manifestVariables: [[key : value], [key : value]]          //optional, default: []
+)
+```
+
+For example, you can refer to the parameters using relative paths (similar to `cf push --vars-file`):
+
+```groovy
+cfManifestSubstituteVariables (
+  script: this,
+  manifestFile: "manifest.yml",
+  manifestVariablesFiles: ["manifest-variables.yml"]
+)
+```
+
+Furthermore, you can also specify variables and their values directly (similar to `cf push --var`):
+
+```groovy
+cfManifestSubstituteVariables (
+  script: this,
+  manifestFile: "manifest.yml",
+  manifestVariablesFiles: ["manifest-variables.yml"],
+  manifestVariables: [[key1 : value1], [key2 : value2]]
+)
+```
+
+If you are using the Cloud Foundry [Create-Service-Push](https://github.com/dawu415/CF-CLI-Create-Service-Push-Plugin) CLI plugin you will most likely also have a `services-manifest.yml` file.
+Also in this file you can specify variable references, that can be resolved from the same variables file, e.g. like this:
+
+```groovy
+// resolve variables in manifest.yml
+cfManifestSubstituteVariables (
+  script: this,
+  manifestFile: "manifest.yml",
+  manifestVariablesFiles: ["manifest-variables.yml"]
+)
+
+// resolve variables in services-manifest.yml from same file.
+cfManifestSubstituteVariables (
+  script: this,
+  manifestFile: "services-manifest.yml",
+  manifestVariablesFiles: ["manifest-variables.yml"]
+)
+```

documentation/docs/steps/checkChangeInDevelopment.md

@@ -4,7 +4,7 @@
 
 ## Prerequisites
 
-* **[Change Management Client 2.0.0 or compatible version](http://central.maven.org/maven2/com/sap/devops/cmclient/dist.cli/)** - available for download on Maven Central.
+* **[Change Management Client 2.0.0 or compatible version](http://central.maven.org/maven2/com/sap/devops/cmclient/dist.cli/)** - available for download on Maven Central. **Note:** This is only required if you don't use a Docker-based environment.
 
 ## ${docGenParameters}
 

documentation/docs/steps/piperPipelineStagePost.md

@@ -1,9 +0,0 @@
-# ${docGenStepName}
-
-## ${docGenDescription}
-
-## ${docGenParameters}
-
-## ${docGenConfiguration}
-
-## ${docJenkinsPluginDependencies}

documentation/docs/steps/transportRequestCreate.md

@@ -4,7 +4,7 @@
 
 ## Prerequisites
 
-* **[Change Management Client 2.0.0 or compatible version](http://central.maven.org/maven2/com/sap/devops/cmclient/dist.cli/)** - available for download on Maven Central.
+* **[Change Management Client 2.0.0 or compatible version](http://central.maven.org/maven2/com/sap/devops/cmclient/dist.cli/)** - available for download on Maven Central. **Note:** This is only required if you don't use a Docker-based environment.
 * Solution Manager version `ST720 SP08` or newer.
 
 ## ${docGenParameters}

documentation/docs/steps/transportRequestRelease.md

@@ -4,7 +4,7 @@
 
 ## Prerequisites
 
-* **[Change Management Client 2.0.0 or compatible version](http://central.maven.org/maven2/com/sap/devops/cmclient/dist.cli/)** - available for download on Maven Central.
+* **[Change Management Client 2.0.0 or compatible version](http://central.maven.org/maven2/com/sap/devops/cmclient/dist.cli/)** - available for download on Maven Central. **Note:** This is only required if you don't use a Docker-based environment.
 
 ## ${docGenParameters}
 

documentation/docs/steps/transportRequestUploadFile.md

@@ -4,7 +4,7 @@
 
 ## Prerequisites
 
-* **[Change Management Client 2.0.0 or compatible version](http://central.maven.org/maven2/com/sap/devops/cmclient/dist.cli/)** - available for download on Maven Central.
+* **[Change Management Client 2.0.0 or compatible version](http://central.maven.org/maven2/com/sap/devops/cmclient/dist.cli/)** - available for download on Maven Central. **Note:** This is only required if you don't use a Docker-based environment.
 
 ## ${docGenParameters}
 

documentation/docs/steps/xsDeploy.md

@@ -0,0 +1,41 @@
+# ${docGenStepName}
+
+## ${docGenDescription}
+
+## ${docGenParameters}
+
+## ${docGenConfiguration}
+
+## ${docJenkinsPluginDependencies}
+
+## Side effects
+
+none
+
+## Example
+
+```groovy
+xsDeploy
+    script: this,
+    mtaPath: 'path/to/archiveFile.mtar',
+    credentialsId: 'my-credentials-id',
+    apiUrl: 'https://example.org/xs',
+    space: 'mySpace',
+    org:: 'myOrg'
+```
+
+Example configuration:
+
+```yaml
+steps:
+  <...>
+  xsDeploy:
+    script: this,
+    mtaPath: path/to/archiveFile.mtar
+    credentialsId: my-credentials-id
+    apiUrl: https://example.org/xs
+    space: mySpace
+    org:: myOrg
+```
+
+[dockerExecute]: ../dockerExecute

documentation/mkdocs.yml

@@ -1,17 +1,43 @@
-site_name: Jenkins 2.0 Pipelines
+site_name: 'Project "Piper": Continuous Delivery for the SAP Ecosystem'
 nav:
     - Home: index.md
     - 'Guided Tour' : guidedtour.md
     - Configuration: configuration.md
+    - 'Pipelines':
+        - 'General purpose pipeline':
+            - 'Introduction': stages/introduction.md
+            - 'Examples': stages/examples.md
+            - 'Stages':
+                - 'Init Stage': stages/init.md
+                - 'Pull-Request Voting Stage': stages/prvoting.md
+                - 'Build Stage': stages/build.md
+                - 'Additional Unit Test Stage': stages/additionalunittests.md
+                - 'Integration Stage': stages/integration.md
+                - 'Acceptance Stage': stages/acceptance.md
+                - 'Security Stage': stages/security.md
+                - 'Performance Stage': stages/performance.md
+                - 'Compliance': stages/compliance.md
+                - 'Confirm Stage': stages/confirm.md
+                - 'Promote Stage': stages/promote.md
+                - 'Release Stage': stages/release.md
+        - 'SAP Cloud SDK pipeline': pipelines/cloud-sdk/introduction.md
+    - 'Scenarios':
+        - 'Build and Deploy Hybrid Applications with Jenkins and SAP Solution Manager': scenarios/changeManagement.md
+        - 'Build and Deploy SAP UI5 or SAP Fiori Applications on SAP Cloud Platform with Jenkins': scenarios/ui5-sap-cp/Readme.md
+        - 'Build and Deploy Applications with Jenkins and the SAP Cloud Application Programming Model': scenarios/CAP_Scenario.md
+        - 'Integrate SAP Cloud Platform Transport Management Into Your CI/CD Pipeline': scenarios/TMS_Extension.md
     - Extensibility: extensibility.md
     - 'Library steps':
         - artifactSetVersion: steps/artifactSetVersion.md
         - batsExecuteTests: steps/batsExecuteTests.md
+        - buildExecute: steps/buildExecute.md
         - checkChangeInDevelopment: steps/checkChangeInDevelopment.md
         - checksPublishResults: steps/checksPublishResults.md
+        - cfManifestSubstituteVariables: steps/cfManifestSubstituteVariables.md
         - cloudFoundryDeploy: steps/cloudFoundryDeploy.md
         - commonPipelineEnvironment: steps/commonPipelineEnvironment.md
         - containerExecuteStructureTests: steps/containerExecuteStructureTests.md
+        - containerPushToRegistry: steps/containerPushToRegistry.md
         - detectExecuteScan: steps/detectExecuteScan.md
         - dockerExecute: steps/dockerExecute.md
         - dockerExecuteOnKubernetes: steps/dockerExecuteOnKubernetes.md
@@ -51,6 +77,7 @@ nav:
         - transportRequestUploadFile: steps/transportRequestUploadFile.md
         - uiVeri5ExecuteTests: steps/uiVeri5ExecuteTests.md
         - whitesourceExecuteScan: steps/whitesourceExecuteScan.md
+        - xsDeploy: steps/xsDeploy.md
     - 'Pipelines':
         - 'General purpose pipeline':
             - 'Introduction': stages/introduction.md
@@ -72,6 +99,7 @@ nav:
         - 'Build and Deploy Hybrid Applications with Jenkins and SAP Solution Manager': scenarios/changeManagement.md
         - 'Build and Deploy SAP UI5 or SAP Fiori Applications on SAP Cloud Platform with Jenkins': scenarios/ui5-sap-cp/Readme.md
         - 'Build and Deploy Applications with Jenkins and the SAP Cloud Application Programming Model': scenarios/CAP_Scenario.md
+        - 'Build and Deploy SAP Fiori Applications for SAP HANA XS Advanced ': scenarios/xsa-deploy/Readme.md
     - Resources:
         - 'Required Plugins': jenkins/requiredPlugins.md
 

go.mod

@@ -0,0 +1,11 @@
+module github.com/SAP/jenkins-library
+
+go 1.13
+
+require (
+	github.com/ghodss/yaml v1.0.0
+	github.com/pkg/errors v0.8.1
+	github.com/spf13/cobra v0.0.5
+	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.2.2
+)

go.sum

@@ -0,0 +1,42 @@
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/config/config.go

@@ -0,0 +1,189 @@
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
+)
+
+// Config defines the structure of the config files
+type Config struct {
+	General map[string]interface{}            `json:"general"`
+	Stages  map[string]map[string]interface{} `json:"stages"`
+	Steps   map[string]map[string]interface{} `json:"steps"`
+}
+
+// StepConfig defines the structure for merged step configuration
+type StepConfig struct {
+	Config map[string]interface{}
+}
+
+// ReadConfig loads config and returns its content
+func (c *Config) ReadConfig(configuration io.ReadCloser) error {
+	defer configuration.Close()
+
+	content, err := ioutil.ReadAll(configuration)
+	if err != nil {
+		return errors.Wrapf(err, "error reading %v", configuration)
+	}
+
+	err = yaml.Unmarshal(content, &c)
+	if err != nil {
+		return NewParseError(fmt.Sprintf("error unmarshalling %q: %v", content, err))
+	}
+	return nil
+}
+
+// GetStepConfig provides merged step configuration using defaults, config, if available
+func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON string, configuration io.ReadCloser, defaults []io.ReadCloser, filters StepFilters, stageName, stepName string) (StepConfig, error) {
+	var stepConfig StepConfig
+	var d PipelineDefaults
+
+	if err := c.ReadConfig(configuration); err != nil {
+		switch err.(type) {
+		case *ParseError:
+			return StepConfig{}, errors.Wrap(err, "failed to parse custom pipeline configuration")
+		default:
+			//ignoring unavailability of config file since considered optional
+		}
+	}
+
+	if err := d.ReadPipelineDefaults(defaults); err != nil {
+		switch err.(type) {
+		case *ParseError:
+			return StepConfig{}, errors.Wrap(err, "failed to parse pipeline default configuration")
+		default:
+			//ignoring unavailability of defaults since considered optional
+		}
+	}
+
+	// first: read defaults & merge general -> steps (-> general -> steps ...)
+	for _, def := range d.Defaults {
+		stepConfig.mixIn(def.General, filters.General)
+		stepConfig.mixIn(def.Steps[stepName], filters.Steps)
+	}
+
+	// second: read config & merge - general -> steps -> stages
+	stepConfig.mixIn(c.General, filters.General)
+	stepConfig.mixIn(c.Steps[stepName], filters.Steps)
+	stepConfig.mixIn(c.Stages[stageName], filters.Stages)
+
+	// third: merge parameters provided via env vars
+	stepConfig.mixIn(envValues(filters.All), filters.All)
+
+	// fourth: if parameters are provided in JSON format merge them
+	if len(paramJSON) != 0 {
+		var params map[string]interface{}
+		json.Unmarshal([]byte(paramJSON), &params)
+		stepConfig.mixIn(params, filters.Parameters)
+	}
+
+	// fifth: merge command line flags
+	if flagValues != nil {
+		stepConfig.mixIn(flagValues, filters.Parameters)
+	}
+
+	return stepConfig, nil
+}
+
+// GetStepConfigWithJSON provides merged step configuration using a provided stepConfigJSON with additional flags provided
+func GetStepConfigWithJSON(flagValues map[string]interface{}, stepConfigJSON string, filters StepFilters) StepConfig {
+	var stepConfig StepConfig
+
+	stepConfigMap := map[string]interface{}{}
+
+	json.Unmarshal([]byte(stepConfigJSON), &stepConfigMap)
+
+	stepConfig.mixIn(stepConfigMap, filters.All)
+
+	// ToDo: mix in parametersJSON
+
+	if flagValues != nil {
+		stepConfig.mixIn(flagValues, filters.Parameters)
+	}
+	return stepConfig
+}
+
+// GetJSON returns JSON representation of an object
+func GetJSON(data interface{}) (string, error) {
+
+	result, err := json.Marshal(data)
+	if err != nil {
+		return "", errors.Wrapf(err, "error marshalling json: %v", err)
+	}
+	return string(result), nil
+}
+
+func envValues(filter []string) map[string]interface{} {
+	vals := map[string]interface{}{}
+	for _, param := range filter {
+		if envVal := os.Getenv("PIPER_" + param); len(envVal) != 0 {
+			vals[param] = os.Getenv("PIPER_" + param)
+		}
+	}
+	return vals
+}
+
+func (s *StepConfig) mixIn(mergeData map[string]interface{}, filter []string) {
+
+	if s.Config == nil {
+		s.Config = map[string]interface{}{}
+	}
+
+	s.Config = filterMap(merge(s.Config, mergeData), filter)
+}
+
+func filterMap(data map[string]interface{}, filter []string) map[string]interface{} {
+	result := map[string]interface{}{}
+
+	if data == nil {
+		data = map[string]interface{}{}
+	}
+
+	for key, value := range data {
+		if len(filter) == 0 || sliceContains(filter, key) {
+			result[key] = value
+		}
+	}
+	return result
+}
+
+func merge(base, overlay map[string]interface{}) map[string]interface{} {
+
+	result := map[string]interface{}{}
+
+	if base == nil {
+		base = map[string]interface{}{}
+	}
+
+	for key, value := range base {
+		result[key] = value
+	}
+
+	for key, value := range overlay {
+		if val, ok := value.(map[string]interface{}); ok {
+			if valBaseKey, ok := base[key].(map[string]interface{}); !ok {
+				result[key] = merge(map[string]interface{}{}, val)
+			} else {
+				result[key] = merge(valBaseKey, val)
+			}
+		} else {
+			result[key] = value
+		}
+	}
+	return result
+}
+
+func sliceContains(slice []string, find string) bool {
+	for _, elem := range slice {
+		if elem == find {
+			return true
+		}
+	}
+	return false
+}

pkg/config/config_test.go

@@ -0,0 +1,253 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type errReadCloser int
+
+func (errReadCloser) Read(p []byte) (n int, err error) {
+	return 0, errors.New("read error")
+}
+
+func (errReadCloser) Close() error {
+	return nil
+}
+
+func TestReadConfig(t *testing.T) {
+
+	var c Config
+
+	t.Run("Success case", func(t *testing.T) {
+
+		myConfig := strings.NewReader("general:\n  generalTestKey: generalTestValue\nsteps:\n  testStep:\n    testStepKey: testStepValue")
+
+		err := c.ReadConfig(ioutil.NopCloser(myConfig)) // NopCloser "no-ops" the closing interface since strings do not need to be closed
+		if err != nil {
+			t.Errorf("Got error although no error expected: %v", err)
+		}
+
+		if c.General["generalTestKey"] != "generalTestValue" {
+			t.Errorf("General config- got: %v, expected: %v", c.General["generalTestKey"], "generalTestValue")
+		}
+
+		if c.Steps["testStep"]["testStepKey"] != "testStepValue" {
+			t.Errorf("Step config - got: %v, expected: %v", c.Steps["testStep"]["testStepKey"], "testStepValue")
+		}
+	})
+
+	t.Run("Read failure", func(t *testing.T) {
+		var rc errReadCloser
+		err := c.ReadConfig(rc)
+		if err == nil {
+			t.Errorf("Got no error although error expected.")
+		}
+	})
+
+	t.Run("Unmarshalling failure", func(t *testing.T) {
+		myConfig := strings.NewReader("general:\n  generalTestKey: generalTestValue\nsteps:\n  testStep:\n\ttestStepKey: testStepValue")
+		err := c.ReadConfig(ioutil.NopCloser(myConfig))
+		if err == nil {
+			t.Errorf("Got no error although error expected.")
+		}
+	})
+
+}
+
+func TestGetStepConfig(t *testing.T) {
+
+	t.Run("Success case", func(t *testing.T) {
+
+		testConfig := `general:
+  p3: p3_general
+  px3: px3_general
+  p4: p4_general
+steps:
+  step1:
+    p4: p4_step
+    px4: px4_step
+    p5: p5_step
+stages:
+  stage1:
+    p5: p5_stage
+    px5: px5_stage
+    p6: p6_stage
+`
+		filters := StepFilters{
+			General:    []string{"p0", "p1", "p2", "p3", "p4"},
+			Steps:      []string{"p0", "p1", "p2", "p3", "p4", "p5"},
+			Stages:     []string{"p0", "p1", "p2", "p3", "p4", "p5", "p6"},
+			Parameters: []string{"p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7"},
+			Env:        []string{"p0", "p1", "p2", "p3", "p4", "p5"},
+		}
+
+		defaults1 := `general:
+  p0: p0_general_default
+  px0: px0_general_default
+  p1: p1_general_default
+steps:
+  step1:
+    p1: p1_step_default
+    px1: px1_step_default
+    p2: p2_step_default
+`
+
+		defaults2 := `general:
+  p2: p2_general_default
+  px2: px2_general_default
+  p3: p3_general_default 
+`
+		paramJSON := `{"p6":"p6_param","p7":"p7_param"}`
+
+		flags := map[string]interface{}{"p7": "p7_flag"}
+
+		var c Config
+		defaults := []io.ReadCloser{ioutil.NopCloser(strings.NewReader(defaults1)), ioutil.NopCloser(strings.NewReader(defaults2))}
+
+		myConfig := ioutil.NopCloser(strings.NewReader(testConfig))
+		stepConfig, err := c.GetStepConfig(flags, paramJSON, myConfig, defaults, filters, "stage1", "step1")
+
+		assert.Equal(t, nil, err, "error occured but none expected")
+
+		t.Run("Config", func(t *testing.T) {
+			expected := map[string]string{
+				"p0": "p0_general_default",
+				"p1": "p1_step_default",
+				"p2": "p2_general_default",
+				"p3": "p3_general",
+				"p4": "p4_step",
+				"p5": "p5_stage",
+				"p6": "p6_param",
+				"p7": "p7_flag",
+			}
+			for k, v := range expected {
+				t.Run(k, func(t *testing.T) {
+					if stepConfig.Config[k] != v {
+						t.Errorf("got: %v, expected: %v", stepConfig.Config[k], v)
+					}
+				})
+			}
+		})
+
+		t.Run("Config not expected", func(t *testing.T) {
+			notExpectedKeys := []string{"px0", "px1", "px2", "px3", "px4", "px5"}
+			for _, p := range notExpectedKeys {
+				t.Run(p, func(t *testing.T) {
+					if stepConfig.Config[p] != nil {
+						t.Errorf("unexpected: %v", p)
+					}
+				})
+			}
+		})
+	})
+
+	t.Run("Failure case config", func(t *testing.T) {
+		var c Config
+		myConfig := ioutil.NopCloser(strings.NewReader("invalid config"))
+		_, err := c.GetStepConfig(nil, "", myConfig, nil, StepFilters{}, "stage1", "step1")
+		assert.EqualError(t, err, "failed to parse custom pipeline configuration: error unmarshalling \"invalid config\": error unmarshaling JSON: json: cannot unmarshal string into Go value of type config.Config", "default error expected")
+	})
+
+	t.Run("Failure case defaults", func(t *testing.T) {
+		var c Config
+		myConfig := ioutil.NopCloser(strings.NewReader(""))
+		myDefaults := []io.ReadCloser{ioutil.NopCloser(strings.NewReader("invalid defaults"))}
+		_, err := c.GetStepConfig(nil, "", myConfig, myDefaults, StepFilters{}, "stage1", "step1")
+		assert.EqualError(t, err, "failed to parse pipeline default configuration: error unmarshalling \"invalid defaults\": error unmarshaling JSON: json: cannot unmarshal string into Go value of type config.Config", "default error expected")
+	})
+
+	//ToDo: test merging of env and parameters/flags
+}
+
+func TestGetStepConfigWithJSON(t *testing.T) {
+
+	filters := StepFilters{All: []string{"key1"}}
+
+	t.Run("Without flags", func(t *testing.T) {
+		sc := GetStepConfigWithJSON(nil, `"key1":"value1","key2":"value2"`, filters)
+
+		if sc.Config["key1"] != "value1" && sc.Config["key2"] == "value2" {
+			t.Errorf("got: %v, expected: %v", sc.Config, StepConfig{Config: map[string]interface{}{"key1": "value1"}})
+		}
+	})
+
+	t.Run("With flags", func(t *testing.T) {
+		flags := map[string]interface{}{"key1": "flagVal1"}
+		sc := GetStepConfigWithJSON(flags, `"key1":"value1","key2":"value2"`, filters)
+		if sc.Config["key1"] != "flagVal1" {
+			t.Errorf("got: %v, expected: %v", sc.Config["key1"], "flagVal1")
+		}
+	})
+}
+
+func TestGetJSON(t *testing.T) {
+
+	t.Run("Success case", func(t *testing.T) {
+		custom := map[string]interface{}{"key1": "value1"}
+		json, err := GetJSON(custom)
+		if err != nil {
+			t.Errorf("Got error although no error expected: %v", err)
+		}
+
+		if json != `{"key1":"value1"}` {
+			t.Errorf("got: %v, expected: %v", json, `{"key1":"value1"}`)
+		}
+
+	})
+	t.Run("Marshalling failure", func(t *testing.T) {
+		_, err := GetJSON(make(chan int))
+		if err == nil {
+			t.Errorf("Got no error although error expected")
+		}
+	})
+}
+
+func TestMerge(t *testing.T) {
+
+	testTable := []struct {
+		Source         map[string]interface{}
+		Filter         []string
+		MergeData      map[string]interface{}
+		ExpectedOutput map[string]interface{}
+	}{
+		{
+			Source:         map[string]interface{}{"key1": "baseValue"},
+			Filter:         []string{},
+			MergeData:      map[string]interface{}{"key1": "overwrittenValue"},
+			ExpectedOutput: map[string]interface{}{"key1": "overwrittenValue"},
+		},
+		{
+			Source:         map[string]interface{}{"key1": "value1"},
+			Filter:         []string{},
+			MergeData:      map[string]interface{}{"key2": "value2"},
+			ExpectedOutput: map[string]interface{}{"key1": "value1", "key2": "value2"},
+		},
+		{
+			Source:         map[string]interface{}{"key1": "value1"},
+			Filter:         []string{"key1"},
+			MergeData:      map[string]interface{}{"key2": "value2"},
+			ExpectedOutput: map[string]interface{}{"key1": "value1"},
+		},
+		{
+			Source:         map[string]interface{}{"key1": map[string]interface{}{"key1_1": "value1"}},
+			Filter:         []string{},
+			MergeData:      map[string]interface{}{"key1": map[string]interface{}{"key1_2": "value2"}},
+			ExpectedOutput: map[string]interface{}{"key1": map[string]interface{}{"key1_1": "value1", "key1_2": "value2"}},
+		},
+	}
+
+	for _, row := range testTable {
+		t.Run(fmt.Sprintf("Merging %v into %v", row.MergeData, row.Source), func(t *testing.T) {
+			stepConfig := StepConfig{Config: row.Source}
+			stepConfig.mixIn(row.MergeData, row.Filter)
+			assert.Equal(t, row.ExpectedOutput, stepConfig.Config, "Mixin  was incorrect")
+		})
+	}
+}

pkg/config/defaults.go

@@ -0,0 +1,40 @@
+package config
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
+)
+
+// PipelineDefaults defines the structure of the pipeline defaults
+type PipelineDefaults struct {
+	Defaults []Config `json:"defaults"`
+}
+
+// ReadPipelineDefaults loads defaults and returns its content
+func (d *PipelineDefaults) ReadPipelineDefaults(defaultSources []io.ReadCloser) error {
+
+	for _, def := range defaultSources {
+
+		defer def.Close()
+
+		var c Config
+		var err error
+
+		content, err := ioutil.ReadAll(def)
+		if err != nil {
+			return errors.Wrapf(err, "error reading %v", def)
+		}
+
+		err = yaml.Unmarshal(content, &c)
+		if err != nil {
+			return NewParseError(fmt.Sprintf("error unmarshalling %q: %v", content, err))
+		}
+
+		d.Defaults = append(d.Defaults, c)
+	}
+	return nil
+}

pkg/config/defaults_test.go

@@ -0,0 +1,53 @@
+package config
+
+import (
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+)
+
+func TestReadPipelineDefaults(t *testing.T) {
+
+	var d PipelineDefaults
+
+	t.Run("Success case", func(t *testing.T) {
+		d0 := strings.NewReader("general:\n  testStepKey1: testStepValue1")
+		d1 := strings.NewReader("general:\n  testStepKey2: testStepValue2")
+		err := d.ReadPipelineDefaults([]io.ReadCloser{ioutil.NopCloser(d0), ioutil.NopCloser(d1)})
+
+		if err != nil {
+			t.Errorf("Got error although no error expected: %v", err)
+		}
+
+		t.Run("Defaults 0", func(t *testing.T) {
+			expected := "testStepValue1"
+			if d.Defaults[0].General["testStepKey1"] != expected {
+				t.Errorf("got: %v, expected: %v", d.Defaults[0].General["testStepKey1"], expected)
+			}
+		})
+
+		t.Run("Defaults 1", func(t *testing.T) {
+			expected := "testStepValue2"
+			if d.Defaults[1].General["testStepKey2"] != expected {
+				t.Errorf("got: %v, expected: %v", d.Defaults[1].General["testStepKey2"], expected)
+			}
+		})
+	})
+
+	t.Run("Read failure", func(t *testing.T) {
+		var rc errReadCloser
+		err := d.ReadPipelineDefaults([]io.ReadCloser{rc})
+		if err == nil {
+			t.Errorf("Got no error although error expected.")
+		}
+	})
+
+	t.Run("Unmarshalling failure", func(t *testing.T) {
+		myConfig := strings.NewReader("general:\n\ttestStepKey: testStepValue")
+		err := d.ReadPipelineDefaults([]io.ReadCloser{ioutil.NopCloser(myConfig)})
+		if err == nil {
+			t.Errorf("Got no error although error expected.")
+		}
+	})
+}

pkg/config/errors.go

@@ -0,0 +1,18 @@
+package config
+
+// ParseError defines an error type for configuration parsing errors
+type ParseError struct {
+	message string
+}
+
+// NewParseError creates a new ParseError
+func NewParseError(message string) *ParseError {
+	return &ParseError{
+		message: message,
+	}
+}
+
+// Error returns the message of the ParseError
+func (e *ParseError) Error() string {
+	return e.message
+}

pkg/config/errors_test.go

@@ -0,0 +1,13 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseError(t *testing.T) {
+	err := NewParseError("Parsing failed")
+
+	assert.Equal(t, "Parsing failed", err.Error())
+}

pkg/config/flags.go

@@ -0,0 +1,43 @@
+package config
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	flag "github.com/spf13/pflag"
+)
+
+// AvailableFlagValues returns all flags incl. values which are available to the command.
+func AvailableFlagValues(cmd *cobra.Command, filters *StepFilters) map[string]interface{} {
+	flagValues := map[string]interface{}{}
+	flags := cmd.Flags()
+	//only check flags where value has been set
+	flags.Visit(func(pflag *flag.Flag) {
+
+		switch pflag.Value.Type() {
+		case "string":
+			flagValues[pflag.Name] = pflag.Value.String()
+		case "stringSlice":
+			flagValues[pflag.Name], _ = flags.GetStringSlice(pflag.Name)
+		case "bool":
+			flagValues[pflag.Name], _ = flags.GetBool(pflag.Name)
+		default:
+			fmt.Printf("Meta data type not set or not known: '%v'\n", pflag.Value.Type())
+			os.Exit(1)
+		}
+		filters.Parameters = append(filters.Parameters, pflag.Name)
+	})
+	return flagValues
+}
+
+// MarkFlagsWithValue marks a flag as changed if value is available for the flag through the step configuration.
+func MarkFlagsWithValue(cmd *cobra.Command, stepConfig StepConfig) {
+	flags := cmd.Flags()
+	flags.VisitAll(func(pflag *flag.Flag) {
+		//mark as available in case default is available or config is available
+		if len(pflag.Value.String()) > 0 || stepConfig.Config[pflag.Name] != nil {
+			pflag.Changed = true
+		}
+	})
+}

pkg/config/flags_test.go

@@ -0,0 +1,67 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAvailableFlagValues(t *testing.T) {
+	var f StepFilters
+
+	var test0 string
+	var test1 string
+	var test2 []string
+	var test3 bool
+
+	var c = &cobra.Command{
+		Use:   "test",
+		Short: "..",
+	}
+
+	c.Flags().StringVar(&test0, "test0", "val0", "Test 0")
+	c.Flags().StringVar(&test1, "test1", "", "Test 1")
+	c.Flags().StringSliceVar(&test2, "test2", []string{}, "Test 2")
+	c.Flags().BoolVar(&test3, "test3", false, "Test 3")
+
+	c.Flags().Set("test1", "val1")
+	c.Flags().Set("test2", "val3_1")
+	c.Flags().Set("test3", "true")
+
+	v := AvailableFlagValues(c, &f)
+
+	if v["test0"] != nil {
+		t.Errorf("expected: 'test0' to be empty but was %v", v["test0"])
+	}
+
+	assert.Equal(t, "val1", v["test1"])
+	assert.Equal(t, []string{"val3_1"}, v["test2"])
+	assert.Equal(t, true, v["test3"])
+
+}
+
+func TestMarkFlagsWithValue(t *testing.T) {
+	var test0 string
+	var test1 string
+	var test2 string
+	var c = &cobra.Command{
+		Use:   "test",
+		Short: "..",
+	}
+	c.Flags().StringVar(&test0, "test0", "val0", "Test 0")
+	c.Flags().StringVar(&test1, "test1", "", "Test 1")
+	c.Flags().StringVar(&test2, "test2", "", "Test 2")
+
+	s := StepConfig{
+		Config: map[string]interface{}{
+			"test2": "val2",
+		},
+	}
+
+	MarkFlagsWithValue(c, s)
+
+	assert.Equal(t, true, c.Flags().Changed("test0"), "default not considered")
+	assert.Equal(t, false, c.Flags().Changed("test1"), "no value: considered as set")
+	assert.Equal(t, true, c.Flags().Changed("test2"), "config not considered")
+}

pkg/config/stepmeta.go

@@ -0,0 +1,139 @@
+package config
+
+import (
+	"io"
+	"io/ioutil"
+
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
+)
+
+// StepData defines the metadata for a step, like step descriptions, parameters, ...
+type StepData struct {
+	Metadata StepMetadata `json:"metadata"`
+	Spec     StepSpec     `json:"spec"`
+}
+
+// StepMetadata defines the metadata for a step, like step descriptions, parameters, ...
+type StepMetadata struct {
+	Name            string `json:"name"`
+	Description     string `json:"description"`
+	LongDescription string `json:"longDescription,omitempty"`
+}
+
+// StepSpec defines the spec details for a step, like step inputs, containers, sidecars, ...
+type StepSpec struct {
+	Inputs StepInputs `json:"inputs"`
+	//	Outputs string `json:"description,omitempty"`
+	Containers []StepContainers `json:"containers,omitempty"`
+	Sidecars   []StepSidecars   `json:"sidecars,omitempty"`
+}
+
+// StepInputs defines the spec details for a step, like step inputs, containers, sidecars, ...
+type StepInputs struct {
+	Parameters []StepParameters `json:"params"`
+	Resources  []StepResources  `json:"resources,omitempty"`
+	Secrets    []StepSecrets    `json:"secrets,omitempty"`
+}
+
+// StepParameters defines the parameters for a step
+type StepParameters struct {
+	Name            string      `json:"name"`
+	Description     string      `json:"description"`
+	LongDescription string      `json:"longDescription,omitempty"`
+	Scope           []string    `json:"scope"`
+	Type            string      `json:"type"`
+	Mandatory       bool        `json:"mandatory,omitempty"`
+	Default         interface{} `json:"default,omitempty"`
+}
+
+// StepResources defines the resources to be provided by the step context, e.g. Jenkins pipeline
+type StepResources struct {
+	Name        string `json:"name"`
+	Description string `json:"description,omitempty"`
+	Type        string `json:"type,omitempty"`
+}
+
+// StepSecrets defines the secrets to be provided by the step context, e.g. Jenkins pipeline
+type StepSecrets struct {
+	Name        string `json:"name"`
+	Description string `json:"description,omitempty"`
+	Type        string `json:"type,omitempty"`
+}
+
+// StepOutputs defines the outputs of a step
+//type StepOutputs struct {
+//	Name          string `json:"name"`
+//}
+
+// StepContainers defines the containers required for a step
+type StepContainers struct {
+	Containers map[string]interface{} `json:"containers"`
+}
+
+// StepSidecars defines any sidears required for a step
+type StepSidecars struct {
+	Sidecars map[string]interface{} `json:"sidecars"`
+}
+
+// StepFilters defines the filter parameters for the different sections
+type StepFilters struct {
+	All        []string
+	General    []string
+	Stages     []string
+	Steps      []string
+	Parameters []string
+	Env        []string
+}
+
+// ReadPipelineStepData loads step definition in yaml format
+func (m *StepData) ReadPipelineStepData(metadata io.ReadCloser) error {
+	defer metadata.Close()
+	content, err := ioutil.ReadAll(metadata)
+	if err != nil {
+		return errors.Wrapf(err, "error reading %v", metadata)
+	}
+
+	err = yaml.Unmarshal(content, &m)
+	if err != nil {
+		return errors.Wrapf(err, "error unmarshalling: %v", err)
+	}
+	return nil
+}
+
+// GetParameterFilters retrieves all scope dependent parameter filters
+func (m *StepData) GetParameterFilters() StepFilters {
+	var filters StepFilters
+	for _, param := range m.Spec.Inputs.Parameters {
+		filters.All = append(filters.All, param.Name)
+		for _, scope := range param.Scope {
+			switch scope {
+			case "GENERAL":
+				filters.General = append(filters.General, param.Name)
+			case "STEPS":
+				filters.Steps = append(filters.Steps, param.Name)
+			case "STAGES":
+				filters.Stages = append(filters.Stages, param.Name)
+			case "PARAMETERS":
+				filters.Parameters = append(filters.Parameters, param.Name)
+			case "ENV":
+				filters.Env = append(filters.Env, param.Name)
+			}
+		}
+	}
+	return filters
+}
+
+// GetContextParameterFilters retrieves all scope dependent parameter filters
+func (m *StepData) GetContextParameterFilters() StepFilters {
+	var filters StepFilters
+	for _, secret := range m.Spec.Inputs.Secrets {
+		filters.All = append(filters.All, secret.Name)
+		filters.General = append(filters.General, secret.Name)
+		filters.Steps = append(filters.Steps, secret.Name)
+		filters.Stages = append(filters.Stages, secret.Name)
+		filters.Parameters = append(filters.Parameters, secret.Name)
+		filters.Env = append(filters.Env, secret.Name)
+	}
+	return filters
+}

pkg/config/stepmeta_test.go

@@ -0,0 +1,266 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"testing"
+)
+
+func TestReadPipelineStepData(t *testing.T) {
+	var s StepData
+
+	t.Run("Success case", func(t *testing.T) {
+		myMeta := strings.NewReader("metadata:\n  name: testIt\nspec:\n  inputs:\n    params:\n      - name: testParamName\n    secrets:\n      - name: testSecret")
+		err := s.ReadPipelineStepData(ioutil.NopCloser(myMeta)) // NopCloser "no-ops" the closing interface since strings do not need to be closed
+
+		if err != nil {
+			t.Errorf("Got error although no error expected: %v", err)
+		}
+
+		t.Run("step name", func(t *testing.T) {
+			if s.Metadata.Name != "testIt" {
+				t.Errorf("Meta name - got: %v, expected: %v", s.Metadata.Name, "testIt")
+			}
+		})
+
+		t.Run("param name", func(t *testing.T) {
+			if s.Spec.Inputs.Parameters[0].Name != "testParamName" {
+				t.Errorf("Step name - got: %v, expected: %v", s.Spec.Inputs.Parameters[0].Name, "testParamName")
+			}
+		})
+
+		t.Run("secret name", func(t *testing.T) {
+			if s.Spec.Inputs.Secrets[0].Name != "testSecret" {
+				t.Errorf("Step name - got: %v, expected: %v", s.Spec.Inputs.Secrets[0].Name, "testSecret")
+			}
+		})
+	})
+
+	t.Run("Read failure", func(t *testing.T) {
+		var rc errReadCloser
+		err := s.ReadPipelineStepData(rc)
+		if err == nil {
+			t.Errorf("Got no error although error expected.")
+		}
+	})
+
+	t.Run("Unmarshalling failure", func(t *testing.T) {
+		myMeta := strings.NewReader("metadata:\n\tname: testIt")
+		err := s.ReadPipelineStepData(ioutil.NopCloser(myMeta))
+		if err == nil {
+			t.Errorf("Got no error although error expected.")
+		}
+	})
+}
+
+func TestGetParameterFilters(t *testing.T) {
+	metadata1 := StepData{
+		Spec: StepSpec{
+			Inputs: StepInputs{
+				Parameters: []StepParameters{
+					{Name: "paramOne", Scope: []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS", "ENV"}},
+					{Name: "paramTwo", Scope: []string{"STEPS", "STAGES", "PARAMETERS", "ENV"}},
+					{Name: "paramThree", Scope: []string{"STAGES", "PARAMETERS", "ENV"}},
+					{Name: "paramFour", Scope: []string{"PARAMETERS", "ENV"}},
+					{Name: "paramFive", Scope: []string{"ENV"}},
+					{Name: "paramSix"},
+				},
+			},
+		},
+	}
+
+	metadata2 := StepData{
+		Spec: StepSpec{
+			Inputs: StepInputs{
+				Parameters: []StepParameters{
+					{Name: "paramOne", Scope: []string{"GENERAL"}},
+					{Name: "paramTwo", Scope: []string{"STEPS"}},
+					{Name: "paramThree", Scope: []string{"STAGES"}},
+					{Name: "paramFour", Scope: []string{"PARAMETERS"}},
+					{Name: "paramFive", Scope: []string{"ENV"}},
+					{Name: "paramSix"},
+				},
+			},
+		},
+	}
+
+	metadata3 := StepData{
+		Spec: StepSpec{
+			Inputs: StepInputs{
+				Parameters: []StepParameters{},
+			},
+		},
+	}
+
+	testTable := []struct {
+		Metadata              StepData
+		ExpectedAll           []string
+		ExpectedGeneral       []string
+		ExpectedStages        []string
+		ExpectedSteps         []string
+		ExpectedParameters    []string
+		ExpectedEnv           []string
+		NotExpectedAll        []string
+		NotExpectedGeneral    []string
+		NotExpectedStages     []string
+		NotExpectedSteps      []string
+		NotExpectedParameters []string
+		NotExpectedEnv        []string
+	}{
+		{
+			Metadata:              metadata1,
+			ExpectedGeneral:       []string{"paramOne"},
+			ExpectedSteps:         []string{"paramOne", "paramTwo"},
+			ExpectedStages:        []string{"paramOne", "paramTwo", "paramThree"},
+			ExpectedParameters:    []string{"paramOne", "paramTwo", "paramThree", "paramFour"},
+			ExpectedEnv:           []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramFive"},
+			ExpectedAll:           []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramFive", "paramSix"},
+			NotExpectedGeneral:    []string{"paramTwo", "paramThree", "paramFour", "paramFive", "paramSix"},
+			NotExpectedSteps:      []string{"paramThree", "paramFour", "paramFive", "paramSix"},
+			NotExpectedStages:     []string{"paramFour", "paramFive", "paramSix"},
+			NotExpectedParameters: []string{"paramFive", "paramSix"},
+			NotExpectedEnv:        []string{"paramSix"},
+			NotExpectedAll:        []string{},
+		},
+		{
+			Metadata:              metadata2,
+			ExpectedGeneral:       []string{"paramOne"},
+			ExpectedSteps:         []string{"paramTwo"},
+			ExpectedStages:        []string{"paramThree"},
+			ExpectedParameters:    []string{"paramFour"},
+			ExpectedEnv:           []string{"paramFive"},
+			ExpectedAll:           []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramFive", "paramSix"},
+			NotExpectedGeneral:    []string{"paramTwo", "paramThree", "paramFour", "paramFive", "paramSix"},
+			NotExpectedSteps:      []string{"paramOne", "paramThree", "paramFour", "paramFive", "paramSix"},
+			NotExpectedStages:     []string{"paramOne", "paramTwo", "paramFour", "paramFive", "paramSix"},
+			NotExpectedParameters: []string{"paramOne", "paramTwo", "paramThree", "paramFive", "paramSix"},
+			NotExpectedEnv:        []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramSix"},
+			NotExpectedAll:        []string{},
+		},
+		{
+			Metadata:           metadata3,
+			ExpectedGeneral:    []string{},
+			ExpectedStages:     []string{},
+			ExpectedSteps:      []string{},
+			ExpectedParameters: []string{},
+			ExpectedEnv:        []string{},
+		},
+	}
+
+	for key, row := range testTable {
+		t.Run(fmt.Sprintf("Metadata%v", key), func(t *testing.T) {
+			filters := row.Metadata.GetParameterFilters()
+			t.Run("General", func(t *testing.T) {
+				for _, val := range filters.General {
+					if !sliceContains(row.ExpectedGeneral, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v to be contained in  %v", val, filters.General)
+					}
+					if sliceContains(row.NotExpectedGeneral, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v NOT to be contained in  %v", val, filters.General)
+					}
+				}
+			})
+			t.Run("Steps", func(t *testing.T) {
+				for _, val := range filters.Steps {
+					if !sliceContains(row.ExpectedSteps, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v to be contained in  %v", val, filters.Steps)
+					}
+					if sliceContains(row.NotExpectedSteps, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v NOT to be contained in  %v", val, filters.Steps)
+					}
+				}
+			})
+			t.Run("Stages", func(t *testing.T) {
+				for _, val := range filters.Stages {
+					if !sliceContains(row.ExpectedStages, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v to be contained in  %v", val, filters.Stages)
+					}
+					if sliceContains(row.NotExpectedStages, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v NOT to be contained in  %v", val, filters.Stages)
+					}
+				}
+			})
+			t.Run("Parameters", func(t *testing.T) {
+				for _, val := range filters.Parameters {
+					if !sliceContains(row.ExpectedParameters, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v to be contained in  %v", val, filters.Parameters)
+					}
+					if sliceContains(row.NotExpectedParameters, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v NOT to be contained in  %v", val, filters.Parameters)
+					}
+				}
+			})
+			t.Run("Env", func(t *testing.T) {
+				for _, val := range filters.Env {
+					if !sliceContains(row.ExpectedEnv, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v to be contained in  %v", val, filters.Env)
+					}
+					if sliceContains(row.NotExpectedEnv, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v NOT to be contained in  %v", val, filters.Env)
+					}
+				}
+			})
+			t.Run("All", func(t *testing.T) {
+				for _, val := range filters.All {
+					if !sliceContains(row.ExpectedAll, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v to be contained in  %v", val, filters.All)
+					}
+					if sliceContains(row.NotExpectedAll, val) {
+						t.Errorf("Creation of parameter filter failed, expected: %v NOT to be contained in  %v", val, filters.All)
+					}
+				}
+			})
+		})
+	}
+}
+
+func TestGetContextParameterFilters(t *testing.T) {
+	metadata1 := StepData{
+		Spec: StepSpec{
+			Inputs: StepInputs{
+				Secrets: []StepSecrets{
+					{Name: "testSecret1", Type: "jenkins"},
+					{Name: "testSecret2", Type: "jenkins"},
+				},
+			},
+		},
+	}
+
+	filters := metadata1.GetContextParameterFilters()
+
+	t.Run("Secrets", func(t *testing.T) {
+		for _, s := range metadata1.Spec.Inputs.Secrets {
+			t.Run("All", func(t *testing.T) {
+				if !sliceContains(filters.All, s.Name) {
+					t.Errorf("Creation of context filter failed, expected: %v to be contained", s.Name)
+				}
+			})
+			t.Run("General", func(t *testing.T) {
+				if !sliceContains(filters.General, s.Name) {
+					t.Errorf("Creation of context filter failed, expected: %v to be contained", s.Name)
+				}
+			})
+			t.Run("Step", func(t *testing.T) {
+				if !sliceContains(filters.Steps, s.Name) {
+					t.Errorf("Creation of context filter failed, expected: %v to be contained", s.Name)
+				}
+			})
+			t.Run("Stages", func(t *testing.T) {
+				if !sliceContains(filters.Steps, s.Name) {
+					t.Errorf("Creation of context filter failed, expected: %v to be contained", s.Name)
+				}
+			})
+			t.Run("Parameters", func(t *testing.T) {
+				if !sliceContains(filters.Parameters, s.Name) {
+					t.Errorf("Creation of context filter failed, expected: %v to be contained", s.Name)
+				}
+			})
+			t.Run("Env", func(t *testing.T) {
+				if !sliceContains(filters.Env, s.Name) {
+					t.Errorf("Creation of context filter failed, expected: %v to be contained", s.Name)
+				}
+			})
+		}
+	})
+}

pom.xml

@@ -12,7 +12,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.sap.cp.jenkins</groupId>
     <artifactId>jenkins-library</artifactId>
-    <version>0.11</version>
+    <version>${revision}</version>
 
     <name>SAP CP Piper Library</name>
     <description>Shared library containing steps and utilities to set up continuous deployment processes for SAP technologies.</description>
@@ -40,6 +40,7 @@
     </pluginRepositories>
 
     <properties>
+        <revision>0-SNAPSHOT</revision>
         <findbugs.skip>true</findbugs.skip>
         <jenkins.version>2.32.3</jenkins.version>
         <pipeline.version>2.5</pipeline.version>
@@ -47,7 +48,6 @@
         <java.level>8</java.level>
     </properties>
 
-
     <dependencies>
 
         <dependency>

resources/default_pipeline_environment.yml

@@ -152,9 +152,12 @@ steps:
   cloudFoundryDeploy:
     cloudFoundry:
       apiEndpoint: 'https://api.cf.eu10.hana.ondemand.com'
+    apiParameters: ''
+    loginParameters: ''
     deployTool: 'cf_native'
     deployType: 'standard'
     keepOldInstance: false
+    cfNativeDeployParameters: ''
     mtaDeployParameters: '-f'
     mtaExtensionDescriptor: ''
     mtaPath: ''
@@ -178,6 +181,12 @@ steps:
     stashContent:
       - 'tests'
     testReportFilePath: 'cst-report.json'
+  cloudFoundryCreateService:
+    cloudFoundry:
+      apiEndpoint: 'https://api.cf.eu10.hana.ondemand.com'
+      serviceManifest: 'service-manifest.yml'
+    dockerImage: 'ppiper/cf-cli'
+    dockerWorkspace: '/home/piper'
   detectExecuteScan:
     detect:
       projectVersion: '1'
@@ -501,7 +510,7 @@ steps:
     dockerImage: 'maven:3.5-jdk-8'
     instance: 'SonarCloud'
     options: []
-    pullRequestProvider: 'github'
+    pullRequestProvider: 'GitHub'
     sonarScannerDownloadUrl: 'https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-3.3.0.1492-linux.zip'
   testsPublishResults:
     failOnError: false
@@ -575,3 +584,14 @@ steps:
     nodeLabel: ''
     stashContent:
       - 'pipelineConfigAndTests'
+  xsDeploy:
+    credentialsId: 'XS'
+    deployIdLogPattern: '^.*"xs bg-deploy -i (.*) -a .*".*$'
+    loginOpts: ''
+    deployOpts: ''
+    docker:
+      dockerImage: ''
+      dockerPullImage: false
+    mode: 'DEPLOY'
+    action: 'NONE'
+    xsSessionFile: '.xsconfig'

src/com/sap/piper/ConfigurationHelper.groovy

@@ -1,7 +1,5 @@
 package com.sap.piper
 
-import com.cloudbees.groovy.cps.NonCPS
-
 @API
 class ConfigurationHelper implements Serializable {
 
@@ -22,6 +20,7 @@ class ConfigurationHelper implements Serializable {
     private Script step
     private String name
     private Map validationResults = null
+    private String dependingOn
 
     private ConfigurationHelper(Script step, Map config){
         this.config = config ?: [:]
@@ -50,7 +49,7 @@ class ConfigurationHelper implements Serializable {
         return mixin(stepConfiguration, filter, compatibleParameters)
     }
 
-    final ConfigurationHelper mixin(Map parameters, Set filter = null, Map compatibleParameters = [:]){
+    ConfigurationHelper mixin(Map parameters, Set filter = null, Map compatibleParameters = [:]){
         if (parameters.size() > 0 && compatibleParameters.size() > 0) {
             parameters = ConfigurationMerger.merge(handleCompatibility(compatibleParameters, parameters), null, parameters)
         }
@@ -87,22 +86,25 @@ class ConfigurationHelper implements Serializable {
         return newConfig
     }
 
-    Map dependingOn(dependentKey){
-        return [
-            mixin: {key ->
-                def parts = tokenizeKey(key)
-                def targetMap = config
-                if(parts.size() > 1) {
-                    key = parts.last()
-                    parts.remove(key)
-                    targetMap = getConfigPropertyNested(config, (parts as Iterable).join(SEPARATOR))
-                }
-                def dependentValue = config[dependentKey]
-                if(targetMap[key] == null && dependentValue && config[dependentValue])
-                    targetMap[key] = config[dependentValue][key]
-                return this
-            }
-        ]
+    ConfigurationHelper mixin(String key){
+        def parts = tokenizeKey(key)
+        def targetMap = config
+        if(parts.size() > 1) {
+            key = parts.last()
+            parts.remove(key)
+            targetMap = getConfigPropertyNested(config, parts.join(SEPARATOR))
+        }
+        def dependentValue = config[dependingOn]
+        if(targetMap[key] == null && dependentValue && config[dependentValue])
+            targetMap[key] = config[dependentValue][key]
+
+        dependingOn = null
+        return this
+    }
+
+    ConfigurationHelper dependingOn(dependentKey){
+        dependingOn = dependentKey
+        return this
     }
 
     ConfigurationHelper addIfEmpty(key, value){
@@ -121,8 +123,6 @@ class ConfigurationHelper implements Serializable {
         return this
     }
 
-    @NonCPS // required because we have a closure in the
-            // method body that cannot be CPS transformed
     Map use(){
         handleValidationFailures()
         MapUtils.traverse(config, { v -> (v instanceof GString) ? v.toString() : v })
@@ -130,8 +130,6 @@ class ConfigurationHelper implements Serializable {
         return MapUtils.deepCopy(config)
     }
 
-
-
     /* private */ def getConfigPropertyNested(key) {
         return getConfigPropertyNested(config, key)
     }
@@ -143,7 +141,7 @@ class ConfigurationHelper implements Serializable {
         if (config[parts.head()] != null) {
 
             if (config[parts.head()] in Map && !parts.tail().isEmpty()) {
-                return getConfigPropertyNested(config[parts.head()], (parts.tail() as Iterable).join(SEPARATOR))
+                return getConfigPropertyNested(config[parts.head()], parts.tail().join(SEPARATOR))
             }
 
             if (config[parts.head()].class == String) {
@@ -193,15 +191,12 @@ class ConfigurationHelper implements Serializable {
         return this
     }
 
-    @NonCPS
     private handleValidationFailures() {
         if(! validationResults) return
         if(validationResults.size() == 1) throw validationResults.values().first()
-        String msg = 'ERROR - NO VALUE AVAILABLE FOR: ' +
-            (validationResults.keySet().stream().collect() as Iterable).join(', ')
+        String msg = 'ERROR - NO VALUE AVAILABLE FOR: ' + validationResults.keySet().join(', ')
         IllegalArgumentException iae = new IllegalArgumentException(msg)
         validationResults.each { e -> iae.addSuppressed(e.value) }
         throw iae
     }
-
 }

src/com/sap/piper/ConfigurationLoader.groovy

@@ -1,30 +1,23 @@
 package com.sap.piper
 
-import com.cloudbees.groovy.cps.NonCPS
-
 @API(deprecated = true)
 class ConfigurationLoader implements Serializable {
-    @NonCPS
     static Map stepConfiguration(script, String stepName) {
         return loadConfiguration(script, 'steps', stepName, ConfigurationType.CUSTOM_CONFIGURATION)
     }
 
-    @NonCPS
     static Map stageConfiguration(script, String stageName) {
         return loadConfiguration(script, 'stages', stageName, ConfigurationType.CUSTOM_CONFIGURATION)
     }
 
-    @NonCPS
     static Map defaultStepConfiguration(script, String stepName) {
         return loadConfiguration(script, 'steps', stepName, ConfigurationType.DEFAULT_CONFIGURATION)
     }
 
-    @NonCPS
     static Map defaultStageConfiguration(script, String stageName) {
         return loadConfiguration(script, 'stages', stageName, ConfigurationType.DEFAULT_CONFIGURATION)
     }
 
-    @NonCPS
     static Map generalConfiguration(script){
         try {
             return script?.commonPipelineEnvironment?.configuration?.general ?: [:]
@@ -33,17 +26,14 @@ class ConfigurationLoader implements Serializable {
         }
     }
 
-    @NonCPS
     static Map defaultGeneralConfiguration(script){
         return DefaultValueCache.getInstance()?.getDefaultValues()?.general ?: [:]
     }
 
-    @NonCPS
     static Map postActionConfiguration(script, String actionName){
         return loadConfiguration(script, 'postActions', actionName, ConfigurationType.CUSTOM_CONFIGURATION)
     }
 
-    @NonCPS
     private static Map loadConfiguration(script, String type, String entryName, ConfigurationType configType){
         switch (configType) {
             case ConfigurationType.CUSTOM_CONFIGURATION:

src/com/sap/piper/ConfigurationMerger.groovy

@@ -1,10 +1,8 @@
 package com.sap.piper
 
-import com.cloudbees.groovy.cps.NonCPS
 
 @API(deprecated = true)
 class ConfigurationMerger {
-    @NonCPS
     static Map merge(Map configs, Set configKeys, Map defaults) {
         Map filteredConfig = configKeys?configs.subMap(configKeys):configs
 
@@ -12,7 +10,6 @@ class ConfigurationMerger {
                               MapUtils.pruneNulls(filteredConfig))
     }
 
-    @NonCPS
     static Map merge(
         Map parameters, Set parameterKeys,
         Map configuration, Set configurationKeys,

src/com/sap/piper/DefaultValueCache.groovy

@@ -2,8 +2,6 @@ package com.sap.piper
 
 import com.sap.piper.MapUtils
 
-import com.cloudbees.groovy.cps.NonCPS
-
 @API
 class DefaultValueCache implements Serializable {
     private static DefaultValueCache instance
@@ -14,7 +12,6 @@ class DefaultValueCache implements Serializable {
         this.defaultValues = defaultValues
     }
 
-    @NonCPS
     static getInstance(){
         return instance
     }
@@ -23,7 +20,6 @@ class DefaultValueCache implements Serializable {
         instance = new DefaultValueCache(defaultValues)
     }
 
-    @NonCPS
     Map getDefaultValues(){
         return defaultValues
     }

src/com/sap/piper/JsonUtils.groovy

@@ -7,6 +7,11 @@ String groovyObjectToPrettyJsonString(object) {
     return groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(object))
 }
 
+@NonCPS
+String groovyObjectToJsonString(object) {
+    return groovy.json.JsonOutput.toJson(object)
+}
+
 @NonCPS
 def jsonStringToGroovyObject(text) {
     return new groovy.json.JsonSlurperClassic().parseText(text)

src/com/sap/piper/MapUtils.groovy

@@ -1,30 +1,25 @@
 package com.sap.piper
 
-import com.cloudbees.groovy.cps.NonCPS
-
 class MapUtils implements Serializable {
-    @NonCPS
     static boolean isMap(object){
         return object in Map
     }
 
-    @NonCPS
     static Map pruneNulls(Map m) {
 
         Map result = [:]
 
         m = m ?: [:]
 
-        for(def e : m.entrySet())
-            if(isMap(e.value))
-                result[e.key] = pruneNulls(e.value)
-            else if(e.value != null)
-                result[e.key] = e.value
+        m.each { key, value ->
+            if(isMap(value))
+                result[key] = pruneNulls(value)
+            else if(value != null)
+                result[key] = value
+        }
         return result
     }
 
-
-    @NonCPS
     static Map merge(Map base, Map overlay) {
 
         Map result = [:]
@@ -33,9 +28,9 @@ class MapUtils implements Serializable {
 
         result.putAll(base)
 
-        for(def e : overlay.entrySet())
-            result[e.key] = isMap(e.value) ? merge(base[e.key], e.value) : e.value
-
+        overlay.each { key, value ->
+            result[key] = isMap(value) ? merge(base[key], value) : value
+        }
         return result
     }
 
@@ -46,18 +41,16 @@ class MapUtils implements Serializable {
      *        in <code>m</code> in a recursive manner.
      * @param strategy Strategy applied to all non-map entries
      */
-    @NonCPS
     static void traverse(Map m, Closure strategy) {
 
         def updates = [:]
-        for(def e : m.entrySet()) {
-            if(isMap(e.value)) {
-                traverse(e.getValue(), strategy)
-            }
-            else {
+        m.each { key, value ->
+            if(isMap(value)) {
+                traverse(value, strategy)
+            } else {
                 // do not update the map while it is traversed. Depending
                 // on the map implementation the behavior is undefined.
-                updates.put(e.getKey(), strategy(e.getValue()))
+                updates.put(key, strategy(value))
             }
         }
         m.putAll(updates)

src/com/sap/piper/SidecarUtils.groovy

@@ -0,0 +1,41 @@
+package com.sap.piper
+
+class SidecarUtils implements Serializable {
+
+    private static Script script
+
+    SidecarUtils(Script script) {
+        this.script = script
+    }
+
+    void waitForSidecarReadyOnDocker(String containerId, String command) {
+        String dockerCommand = "docker exec ${containerId} ${command}"
+        waitForSidecarReady(dockerCommand)
+    }
+
+    void waitForSidecarReadyOnKubernetes(String containerName, String command) {
+        script.container(name: containerName) {
+            waitForSidecarReady(command)
+        }
+    }
+
+    void waitForSidecarReady(String command) {
+        int sleepTimeInSeconds = 10
+        int timeoutInSeconds = 5 * 60
+        int maxRetries = timeoutInSeconds / sleepTimeInSeconds
+        int retries = 0
+        while (true) {
+            script.echo "Waiting for sidecar container"
+            String status = script.sh script: command, returnStatus: true
+            if (status == "0") {
+                return
+            }
+            if (retries > maxRetries) {
+                script.error("Timeout while waiting for sidecar container to be ready")
+            }
+
+            sleep sleepTimeInSeconds
+            retries++
+        }
+    }
+}

src/com/sap/piper/Utils.groovy

@@ -2,7 +2,7 @@ package com.sap.piper
 
 import com.cloudbees.groovy.cps.NonCPS
 import com.sap.piper.analytics.Telemetry
-import groovy.text.SimpleTemplateEngine
+import groovy.text.GStringTemplateEngine
 
 import java.nio.charset.StandardCharsets
 import java.security.MessageDigest
@@ -109,7 +109,7 @@ void pushToSWA(Map parameters, Map config) {
 
 @NonCPS
 static String fillTemplate(String templateText, Map binding) {
-    def engine = new SimpleTemplateEngine()
+    def engine = new GStringTemplateEngine()
     String result = engine.createTemplate(templateText).make(binding)
     return result
 }

src/com/sap/piper/WhitesourceConfigurationHelper.groovy

@@ -23,7 +23,10 @@ class WhitesourceConfigurationHelper implements Serializable {
             ]
         }
         if(config.verbose)
-            mapping += [name: 'log.level', value: 'debug']
+            mapping += [
+                [name: 'log.level', value: 'debug'],
+                [name: 'log.files.level', value: 'debug']
+            ]
 
         mapping += [
             [name: 'apiKey', value: config.whitesource.orgToken, force: true],

src/com/sap/piper/analytics/Telemetry.groovy

@@ -12,7 +12,6 @@ class Telemetry implements Serializable{
 
     protected Telemetry(){}
 
-    @NonCPS
     protected static Telemetry getInstance(){
         if(!instance) {
             instance = new Telemetry()

src/com/sap/piper/variablesubstitution/DebugHelper.groovy

@@ -0,0 +1,53 @@
+package com.sap.piper.variablesubstitution
+
+/**
+ * Very simple debug helper. Declared as a Field
+ * and passed the configuration with a call to `setConfig(Map config)`
+ * in the body of a `call(...)` block, once
+ * the configuration is available.
+ *
+ * If `config.verbose` is set to `true` a message
+ * issued with `debug(String)` will be logged. Otherwise it will silently be omitted.
+ */
+class DebugHelper {
+    /**
+     * The script which will be used to echo debug messages.
+     */
+    private Script script
+    /**
+     * The configuration which will be scanned for a `verbose` flag.
+     * Only if this is true, will debug messages be written.
+     */
+    private Map config
+
+    /**
+     * Creates a new instance using the given script to issue `echo` commands.
+     * The given config's `verbose` flag will decide if a message will be logged or not.
+     * @param script - the script whose `echo` command will be used.
+     * @param config - the configuration whose `verbose` flag is inspected before logging debug statements.
+     */
+    DebugHelper(Script script, Map config) {
+        if(!script) {
+            throw new IllegalArgumentException("[DebugHelper] Script parameter must not be null.")
+        }
+
+        if(!config) {
+            throw new IllegalArgumentException("[DebugHelper] Config map parameter must not be null.")
+        }
+
+        this.script = script
+        this.config = config
+    }
+
+    /**
+     * Logs a debug message if a configuration
+     * indicates that the `verbose` flag
+     * is set to `true`
+     * @param message - the message to log.
+     */
+    void debug(String message) {
+        if(config.verbose) {
+            script.echo message
+        }
+    }
+}

src/com/sap/piper/variablesubstitution/ExecutionContext.groovy

@@ -0,0 +1,18 @@
+package com.sap.piper.variablesubstitution
+
+/**
+ * A simple class to capture context information
+ * of the execution of yamlSubstituteVariables.
+ */
+class ExecutionContext {
+    /**
+     * Property indicating if the execution
+     * of yamlSubstituteVariables actually
+     * substituted any variables at all.
+     *
+     * Does NOT indicate that ALL variables were
+     * actually replaced. If set to true, if just indicates
+     * that some or all variables have been replaced.
+     */
+    Boolean variablesReplaced = false
+}

src/com/sap/piper/variablesubstitution/YamlUtils.groovy

@@ -0,0 +1,225 @@
+package com.sap.piper.variablesubstitution
+
+import hudson.AbortException
+
+/**
+ * A utility class for Yaml data.
+ * Deals with the substitution of variables within Yaml objects.
+ */
+class YamlUtils implements Serializable {
+
+    private final DebugHelper logger
+    private final Script script
+
+    /**
+     * Creates a new utils instance for the given script.
+     * @param script - the script which will be used to call pipeline steps.
+     * @param logger - an optional debug helper to print debug messages.
+     */
+    YamlUtils(Script script, DebugHelper logger = null) {
+        if(!script) {
+            throw new IllegalArgumentException("[YamlUtils] Script must not be null.")
+        }
+        this.script = script
+        this.logger = logger
+    }
+
+    /**
+     * Substitutes variables references in a given input Yaml object with values that are read from the
+     * passed variables Yaml object. Variables may be of primitive or complex types.
+     * The format of variable references follows [Cloud Foundry standards](https://docs.cloudfoundry.org/devguide/deploy-apps/manifest-attributes.html#variable-substitution)
+     *
+     * @param inputYaml - the input Yaml data as `Object`. Can be either of type `Map` or `List`.
+     * @param variablesYaml - the variables Yaml data as `Object`. Can be either of type `Map` or `List` and should
+     *  contain variables names and values to replace variable references contained in `inputYaml`.
+     * @param context - an `ExecutionContext` that can be used to query whether the script actually replaced any variables.
+     * @return the YAML object graph of substituted data.
+     */
+    Object substituteVariables(Object inputYaml, Object variablesYaml, ExecutionContext context = null) {
+        if (!inputYaml) {
+            throw new IllegalArgumentException("[YamlUtils] Input Yaml data must not be null or empty.")
+        }
+
+        if (!variablesYaml) {
+            throw new IllegalArgumentException("[YamlUtils] Variables Yaml data must not be null or empty.")
+        }
+
+        return substitute(inputYaml, variablesYaml, context)
+    }
+
+    /**
+     * Recursively substitutes all variables inside the object tree of the manifest YAML.
+     * @param manifestNode - the manifest YAML to replace variables in.
+     * @param variablesData - the variables values.
+     * @param context - an execution context that can be used to query if any variables were replaced.
+     * @return a YAML object graph which has all variables replaced.
+     */
+    private Object substitute(Object manifestNode, Object variablesData, ExecutionContext context) {
+        Map<String, Object> variableSubstitutes = getVariableSubstitutes(variablesData)
+
+        if (containsVariableReferences(manifestNode)) {
+
+            Object complexResult = null
+            String stringNode = manifestNode as String
+            Map<String, String> referencedVariables = getReferencedVariables(stringNode)
+            referencedVariables.each { referencedVariable ->
+                String referenceToReplace = referencedVariable.getKey()
+                String referenceName = referencedVariable.getValue()
+                Object substitute = variableSubstitutes.get(referenceName)
+
+                if (null == substitute) {
+                    logger?.debug("[YamlUtils] WARNING - Found variable reference ${referenceToReplace} in input Yaml but no variable value to replace it with Leaving it unresolved. Check your variables Yaml data and make sure the variable is properly declared.")
+                    return manifestNode
+                }
+
+                script.echo "[YamlUtils] Replacing: ${referenceToReplace} with ${substitute}"
+
+                if(isSingleVariableReference(stringNode)) {
+                    logger?.debug("[YamlUtils] Node ${stringNode} is SINGLE variable reference. Substitute type is: ${substitute.getClass().getName()}")
+                    // if the string node we need to do replacements for is
+                    // a reference to a single variable, i.e. should be replaced
+                    // entirely with the variable value, we replace the entire node
+                    // with the variable's value (which can possibly be a complex type).
+                    complexResult = substitute
+                }
+                else {
+                    logger?.debug("[YamlUtils] Node ${stringNode} is multi-variable reference or contains additional string constants. Substitute type is: ${substitute.getClass().getName()}")
+                    // if the string node we need to do replacements for contains various
+                    // variable references or a variable reference and constant string additions
+                    // we do a string replacement of the variables inside the node.
+                    String regex = "\\(\\(${referenceName}\\)\\)"
+                    stringNode = stringNode.replaceAll(regex, substitute as String)
+                }
+            }
+
+            if (context) {
+                context.variablesReplaced = true // remember that variables were found in the YAML file that have been replaced.
+            }
+
+            return complexResult ?: stringNode
+        }
+        else if (manifestNode instanceof List) {
+            List<Object> listNode = manifestNode as List<Object>
+            // This copy is only necessary, since Jenkins executes Groovy using
+            // CPS (https://wiki.jenkins.io/display/JENKINS/Pipeline+CPS+method+mismatches)
+            // and has issues with closures in Java 8 lambda expressions. Otherwise we could replace
+            // entries of the list in place (using replaceAll(lambdaExpression))
+            List<Object> copy = new ArrayList<>()
+            listNode.each { entry ->
+                copy.add(substitute(entry, variableSubstitutes, context))
+            }
+            return copy
+        }
+        else if(manifestNode instanceof Map) {
+            Map<String, Object> mapNode = manifestNode as Map<String, Object>
+            // This copy is only necessary to avoid immutability errors reported by Jenkins
+            // runtime environment.
+            Map<String, Object> copy = new HashMap<>()
+            mapNode.entrySet().each { entry ->
+                copy.put(entry.getKey(), substitute(entry.getValue(), variableSubstitutes, context))
+            }
+            return copy
+        }
+        else {
+            logger?.debug("[YamlUtils] Found data type ${manifestNode.getClass().getName()} that needs no substitute. Value: ${manifestNode}")
+            return manifestNode
+        }
+    }
+
+    /**
+     * Turns the parsed variables Yaml data into a
+     * single map. Takes care of multiple YAML sections (separated by ---) if they are found and flattens them into a single
+     * map if necessary.
+     * @param variablesYamlData - the variables data as a Yaml object.
+     * @return the `Map` of variable names mapped to their substitute values.
+     */
+    private Map<String, Object> getVariableSubstitutes(Object variablesYamlData) {
+
+        if(variablesYamlData instanceof List) {
+            return flattenVariablesFileData(variablesYamlData as List)
+        }
+        else if (variablesYamlData instanceof Map) {
+            return variablesYamlData as Map<String, Object>
+        }
+        else {
+            // should never happen (hopefully...)
+            throw new AbortException("[YamlUtils] Found unsupported data type of variables file after parsing YAML. Expected either List or Map. Got: ${variablesYamlData.getClass().getName()}.")
+        }
+    }
+
+    /**
+     * Flattens a list of Yaml sections (which are deemed to be key-value mappings of variable names and values)
+     * to a single map. In case multiple Yaml sections contain the same key, values will be overridden and the result
+     * will be undefined.
+     * @param variablesYamlData - the `List` of Yaml objects of the different sections.
+     * @return the `Map` of variable substitute mappings.
+     */
+    private Map<String, Object> flattenVariablesFileData(List<Map<String, Object>> variablesYamlData) {
+        Map<String, Object> substitutes = new HashMap<>()
+        variablesYamlData.each { map ->
+            map.entrySet().each { entry ->
+                substitutes.put(entry.key, entry.value)
+            }
+        }
+        return substitutes
+    }
+
+    /**
+     * Returns true, if the given object node contains variable references.
+     * @param node - the object-typed value to check for variable references.
+     * @return `true`, if this node references at least one variable, `false` otherwise.
+     */
+    private boolean containsVariableReferences(Object node) {
+        if(!(node instanceof String)) {
+            // variable references can only be contained in
+            // string nodes.
+            return false
+        }
+        String stringNode = node as String
+        return stringNode.contains("((") && stringNode.contains("))")
+    }
+
+    /**
+     * Returns true, if and only if the entire node passed in as a parameter
+     * is a variable reference. Returns false if the node references multiple
+     * variables or if the node embeds the variable reference inside of a constant
+     * string surrounding, e.g. `This-text-has-((numberOfWords))-words`.
+     * @param node - the node to check.
+     * @return `true` if the node is a single variable reference. `false` otherwise.
+     */
+    private boolean isSingleVariableReference(String node) {
+        // regex matching only if the entire node is a reference. (^ = matches start of word, $ = matches end of word)
+        String regex = '^\\(\\([\\d\\w-]*\\)\\)$' // use single quote not to have to escape $ (interpolation) sign.
+        List<String> matches = node.findAll(regex)
+        return (matches != null && !matches.isEmpty())
+    }
+
+    /**
+     * Returns a map of variable references (including braces) to plain variable names referenced in the given `String`.
+     * The keys of the map are the variable references, the values are the names of the referenced variables.
+     * @param value - the value to look for variable references in.
+     * @return the `Map` of names of referenced variables.
+     */
+    private Map<String, String> getReferencedVariables(String value) {
+        Map<String, String> referencesNamesMap = new HashMap<>()
+        List<String> variableReferences = value.findAll("\\(\\([\\d\\w-]*\\)\\)") // find all variables in braces, e.g. ((my-var_05))
+
+        variableReferences.each { reference ->
+            referencesNamesMap.put(reference, getPlainVariableName(reference))
+        }
+
+        return referencesNamesMap
+    }
+
+    /**
+     * Expects a variable reference (including braces) as input and returns the plain name
+     * (by stripping braces) of the variable. E.g. input: `((my_var-04))`, output: `my_var-04`
+     * @param variableReference - the variable reference including braces.
+     * @return the plain variable name
+     */
+    private String getPlainVariableName(String variableReference) {
+        String result = variableReference.replace("((", "")
+        result = result.replace("))", "")
+        return result
+    }
+}

test/groovy/CfManifestSubstituteVariablesTest.groovy

@@ -0,0 +1,600 @@
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.ExpectedException
+import org.junit.rules.RuleChain
+import util.*
+
+import static org.junit.Assert.*
+import static util.JenkinsWriteYamlRule.DATA
+import static util.JenkinsWriteYamlRule.SERIALIZED_YAML
+
+public class CfManifestSubstituteVariablesTest extends BasePiperTest {
+
+    private JenkinsStepRule script = new JenkinsStepRule(this)
+    private JenkinsReadYamlRule readYamlRule = new JenkinsReadYamlRule(this)
+    private JenkinsWriteYamlRule writeYamlRule = new JenkinsWriteYamlRule(this)
+    private JenkinsErrorRule errorRule = new JenkinsErrorRule(this)
+    private JenkinsEnvironmentRule environmentRule = new JenkinsEnvironmentRule(this)
+    private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this)
+    private ExpectedException expectedExceptionRule = ExpectedException.none()
+    private JenkinsFileExistsRule fileExistsRule = new JenkinsFileExistsRule(this, [])
+
+    @Rule
+    public RuleChain rules = Rules
+        .getCommonRules(this)
+        .around(fileExistsRule)
+        .around(readYamlRule)
+        .around(writeYamlRule)
+        .around(errorRule)
+        .around(environmentRule)
+        .around(loggingRule)
+        .around(script)
+        .around(expectedExceptionRule)
+
+    @Before
+    public void setup() {
+        readYamlRule.registerYaml("manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/manifest.yml")))
+                    .registerYaml("manifest-variables.yml", new FileInputStream(new File("test/resources/variableSubstitution/manifest-variables.yml")))
+                    .registerYaml("test/resources/variableSubstitution/manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/manifest.yml")))
+                    .registerYaml("test/resources/variableSubstitution/manifest-variables.yml", new FileInputStream(new File("test/resources/variableSubstitution/manifest-variables.yml")))
+                    .registerYaml("test/resources/variableSubstitution/invalid_manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/invalid_manifest.yml")))
+                    .registerYaml("test/resources/variableSubstitution/novars_manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/novars_manifest.yml")))
+                    .registerYaml("test/resources/variableSubstitution/multi_manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/multi_manifest.yml")))
+                    .registerYaml("test/resources/variableSubstitution/datatypes_manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/datatypes_manifest.yml")))
+                    .registerYaml("test/resources/variableSubstitution/datatypes_manifest-variables.yml", new FileInputStream(new File("test/resources/variableSubstitution/datatypes_manifest-variables.yml")))
+                    .registerYaml("test/resources/variableSubstitution/manifest-variables-conflicting.yml", new FileInputStream(new File("test/resources/variableSubstitution/manifest-variables-conflicting.yml")))
+    }
+
+    @Test
+    public void substituteVariables_SkipsExecution_If_ManifestNotPresent() throws Exception {
+        String manifestFileName = "nonexistent/manifest.yml"
+        String variablesFileName = "nonexistent/manifest-variables.yml"
+        List<String> variablesFiles = [ variablesFileName ]
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Could not find YAML file at ${manifestFileName}. Skipping variable substitution.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, script: nullScript
+
+        //Check that nothing was written
+        assertNull(writeYamlRule.files[manifestFileName])
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_SkipsExecution_If_VariablesFileNotPresent() throws Exception {
+        String manifestFileName = "test/resources/variableSubstitution/manifest.yml"
+        String variablesFileName = "nonexistent/manifest-variables.yml"
+        List<String> variablesFiles = [ variablesFileName ]
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Did not find manifest variable substitution file at ${variablesFileName}.")
+        expectedExceptionRule.expect(hudson.AbortException)
+        expectedExceptionRule.expectMessage("[CFManifestSubstituteVariables] Could not find all given manifest variable substitution files. Make sure all files given as manifestVariablesFiles exist.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, script: nullScript
+
+        //Check that nothing was written
+        assertNull(writeYamlRule.files[manifestFileName])
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_Throws_If_manifestInvalid() throws Exception {
+        String manifestFileName = "test/resources/variableSubstitution/invalid_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/invalid_manifest.yml"
+        List<String> variablesFiles = [ variablesFileName ]
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+        fileExistsRule.registerExistingFile(variablesFileName)
+
+        //check that exception is thrown and that it has the correct message.
+        expectedExceptionRule.expect(org.yaml.snakeyaml.scanner.ScannerException)
+        expectedExceptionRule.expectMessage("found character '%' that cannot start any token. (Do not use % for indentation)")
+
+        loggingRule.expect("[CFManifestSubstituteVariables] Could not load manifest file at ${manifestFileName}.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, script: nullScript
+
+        //Check that nothing was written
+        assertNull(writeYamlRule.files[manifestFileName])
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_Throws_If_manifestVariablesInvalid() throws Exception {
+        String manifestFileName = "test/resources/variableSubstitution/manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/invalid_manifest.yml"
+        List<String> variablesFiles = [ variablesFileName ]
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+        fileExistsRule.registerExistingFile(variablesFileName)
+
+        //check that exception is thrown and that it has the correct message.
+        expectedExceptionRule.expect(org.yaml.snakeyaml.scanner.ScannerException)
+        expectedExceptionRule.expectMessage("found character '%' that cannot start any token. (Do not use % for indentation)")
+
+        loggingRule.expect("[CFManifestSubstituteVariables] Could not load manifest variables file at ${variablesFileName}")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, script: nullScript
+
+        //Check that nothing was written
+        assertNull(writeYamlRule.files[manifestFileName])
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_UsesDefaultFileName_If_NoManifestSpecified() throws Exception {
+        // In this test, we check that the implementation will resort to the default manifest file name.
+        // Since the file is not present, the implementation should stop, but the log should indicate that the
+        // the default file name was used.
+
+        String manifestFileName = "manifest.yml" // default name should be chosen.
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Could not find YAML file at ${manifestFileName}. Skipping variable substitution.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables script: nullScript
+
+        //Check that nothing was written
+        assertNull(writeYamlRule.files[manifestFileName])
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_UsesDefaultFileName_If_NoVariablesFileSpecified() throws Exception {
+        // In this test, we check that the implementation will resort to the default manifest _variables_ file name.
+        // Since the file is not present, the implementation should stop, but the log should indicate that the
+        // the default file name was used.
+
+        String manifestFileName = "test/resources/variableSubstitution/manifest.yml"
+        String variablesFileName = "manifest-variables.yml" // default file name that should be chosen.
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Did not find manifest variable substitution file at ${variablesFileName}.")
+                   .expect("[CFManifestSubstituteVariables] Could not find any default manifest variable substitution file at ${variablesFileName}, and no manifest variables list was specified. Skipping variable substitution.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, script: nullScript
+
+        //Check that nothing was written
+        assertNull(writeYamlRule.files[manifestFileName])
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_ReplacesVariablesProperly_InSingleYamlFiles() throws Exception {
+        String manifestFileName = "test/resources/variableSubstitution/manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/manifest-variables.yml"
+        List<String> variablesFiles = [ variablesFileName ]
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+        fileExistsRule.registerExistingFile(variablesFileName)
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Loaded manifest at ${manifestFileName}!")
+                   .expect("[CFManifestSubstituteVariables] Loaded variables file at ${variablesFileName}!")
+                   .expect("[CFManifestSubstituteVariables] Replaced variables in ${manifestFileName}.")
+                   .expect("[CFManifestSubstituteVariables] Wrote output file (with variables replaced) at ${manifestFileName}.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, script: nullScript
+
+
+        String yamlStringAfterReplacement = writeYamlRule.files[manifestFileName].get(SERIALIZED_YAML) as String
+        Map<String, Object> manifestDataAfterReplacement = writeYamlRule.files[manifestFileName].get(DATA)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        // check that there are no unresolved variables left.
+        assertAllVariablesReplaced(yamlStringAfterReplacement)
+
+        // check that resolved variables have expected values
+        assertCorrectVariableResolution(manifestDataAfterReplacement)
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    private void assertAllVariablesReplaced(String yamlStringAfterReplacement) {
+        assertFalse(yamlStringAfterReplacement.contains("(("))
+        assertFalse(yamlStringAfterReplacement.contains("))"))
+    }
+
+    private void assertCorrectVariableResolution(Map<String, Object> manifestDataAfterReplacement) {
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("name").equals("uniquePrefix-catalog-service-odatav2-0.0.1"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("routes").get(0).get("route").equals("uniquePrefix-catalog-service-odatav2-001.cfapps.eu10.hana.ondemand.com"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("services").get(0).equals("uniquePrefix-catalog-service-odatav2-xsuaa"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("services").get(1).equals("uniquePrefix-catalog-service-odatav2-hana"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("xsuaa-instance-name").equals("uniquePrefix-catalog-service-odatav2-xsuaa"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("db_service_instance_name").equals("uniquePrefix-catalog-service-odatav2-hana"))
+    }
+
+    @Test
+    public void substituteVariables_ReplacesVariablesProperly_InMultiYamlFiles() throws Exception {
+        String manifestFileName = "test/resources/variableSubstitution/multi_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/manifest-variables.yml"
+        List<String> variablesFiles = [ variablesFileName ]
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+        fileExistsRule.registerExistingFile(variablesFileName)
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Loaded manifest at ${manifestFileName}!")
+                   .expect("[CFManifestSubstituteVariables] Loaded variables file at ${variablesFileName}!")
+                   .expect("[CFManifestSubstituteVariables] Replaced variables in ${manifestFileName}.")
+                   .expect("[CFManifestSubstituteVariables] Wrote output file (with variables replaced) at ${manifestFileName}.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, script: nullScript
+
+
+        String yamlStringAfterReplacement = writeYamlRule.files[manifestFileName].get(SERIALIZED_YAML) as String
+        List<Object> manifestDataAfterReplacement = writeYamlRule.files[manifestFileName].get(DATA)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        // check that there are no unresolved variables left.
+        assertAllVariablesReplaced(yamlStringAfterReplacement)
+
+        //check that result still is a multi-YAML file.
+        assertEquals("Dumped YAML after replacement should still be a multi-YAML file.",2, manifestDataAfterReplacement.size())
+
+        // check that resolved variables have expected values
+        manifestDataAfterReplacement.each { yaml ->
+            assertCorrectVariableResolution(yaml as Map<String, Object>)
+        }
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_ReplacesVariablesFromListProperly_whenNoManifestVariablesFilesGiven_InMultiYamlFiles() throws Exception {
+        // This test replaces variables in multi-yaml files from a list of specified variables
+        // the the user has not specified any variable substitution files list.
+
+        String manifestFileName = "test/resources/variableSubstitution/multi_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/manifest-variables.yml"
+
+        List<Map<String, Object>> variablesList = [
+                                                    ["unique-prefix" : "uniquePrefix"],
+                                                    ["xsuaa-instance-name" : "uniquePrefix-catalog-service-odatav2-xsuaa"],
+                                                    ["hana-instance-name" : "uniquePrefix-catalog-service-odatav2-hana"]
+                                                  ]
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+        fileExistsRule.registerExistingFile(variablesFileName)
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Loaded manifest at ${manifestFileName}!")
+            //.expect("[CFManifestSubstituteVariables] Loaded variables file at ${variablesFileName}!")
+            .expect("[CFManifestSubstituteVariables] Replaced variables in ${manifestFileName}.")
+            .expect("[CFManifestSubstituteVariables] Wrote output file (with variables replaced) at ${manifestFileName}.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariables: variablesList, script: nullScript
+
+        String yamlStringAfterReplacement = writeYamlRule.files[manifestFileName].get(SERIALIZED_YAML) as String
+        List<Object> manifestDataAfterReplacement = writeYamlRule.files[manifestFileName].get(DATA)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        // check that there are no unresolved variables left.
+        assertAllVariablesReplaced(yamlStringAfterReplacement)
+
+        //check that result still is a multi-YAML file.
+        assertEquals("Dumped YAML after replacement should still be a multi-YAML file.",2, manifestDataAfterReplacement.size())
+
+        // check that resolved variables have expected values
+        manifestDataAfterReplacement.each { yaml ->
+            assertCorrectVariableResolution(yaml as Map<String, Object>)
+        }
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_ReplacesVariablesFromListProperly_whenEmptyManifestVariablesFilesGiven_InMultiYamlFiles() throws Exception {
+        // This test replaces variables in multi-yaml files from a list of specified variables
+        // the the user has specified an empty list of variable substitution files.
+
+        String manifestFileName = "test/resources/variableSubstitution/multi_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/manifest-variables.yml"
+        List<String> variablesFiles = []
+
+        List<Map<String, Object>> variablesList = [
+            ["unique-prefix" : "uniquePrefix"],
+            ["xsuaa-instance-name" : "uniquePrefix-catalog-service-odatav2-xsuaa"],
+            ["hana-instance-name" : "uniquePrefix-catalog-service-odatav2-hana"]
+        ]
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+        fileExistsRule.registerExistingFile(variablesFileName)
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Loaded manifest at ${manifestFileName}!")
+        //.expect("[CFManifestSubstituteVariables] Loaded variables file at ${variablesFileName}!")
+            .expect("[CFManifestSubstituteVariables] Replaced variables in ${manifestFileName}.")
+            .expect("[CFManifestSubstituteVariables] Wrote output file (with variables replaced) at ${manifestFileName}.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, manifestVariables: variablesList, script: nullScript
+
+        String yamlStringAfterReplacement = writeYamlRule.files[manifestFileName].get(SERIALIZED_YAML) as String
+        List<Object> manifestDataAfterReplacement = writeYamlRule.files[manifestFileName].get(DATA)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        // check that there are no unresolved variables left.
+        assertAllVariablesReplaced(yamlStringAfterReplacement)
+
+        //check that result still is a multi-YAML file.
+        assertEquals("Dumped YAML after replacement should still be a multi-YAML file.",2, manifestDataAfterReplacement.size())
+
+        // check that resolved variables have expected values
+        manifestDataAfterReplacement.each { yaml ->
+            assertCorrectVariableResolution(yaml as Map<String, Object>)
+        }
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_SkipsExecution_If_NoVariablesInManifest() throws Exception {
+        // This test makes sure that, if no variables are found in a manifest that need
+        // to be replaced, the execution is eventually skipped and the manifest remains
+        // untouched.
+
+        String manifestFileName = "test/resources/variableSubstitution/novars_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/manifest-variables.yml"
+        List<String> variablesFiles = [ variablesFileName ]
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+        fileExistsRule.registerExistingFile(variablesFileName)
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Loaded manifest at ${manifestFileName}!")
+                   .expect("[CFManifestSubstituteVariables] Loaded variables file at ${variablesFileName}!")
+                   .expect("[CFManifestSubstituteVariables] No variables were found or could be replaced in ${manifestFileName}. Skipping variable substitution.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, script: nullScript
+
+        //Check that nothing was written
+        assertNull(writeYamlRule.files[manifestFileName])
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_SupportsAllDataTypes() throws Exception {
+        // This test makes sure that, all datatypes supported by YAML are also
+        // properly substituted by the substituteVariables step.
+        // In particular this includes variables of type:
+        // Integer, Boolean, String, Float and inline JSON documents (which are parsed as multi-line strings)
+        // and complex types (like other YAML objects).
+        // The test also checks the differing behaviour when substituting nodes that only consist of a
+        // variable reference and nodes that contains several variable references or additional string constants.
+
+        String manifestFileName = "test/resources/variableSubstitution/datatypes_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/datatypes_manifest-variables.yml"
+        List<String> variablesFiles = [ variablesFileName ]
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+        fileExistsRule.registerExistingFile(variablesFileName)
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Loaded manifest at ${manifestFileName}!")
+                   .expect("[CFManifestSubstituteVariables] Loaded variables file at ${variablesFileName}!")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, script: nullScript
+
+        String yamlStringAfterReplacement = writeYamlRule.files[manifestFileName].get(SERIALIZED_YAML) as String
+        Map<String, Object> manifestDataAfterReplacement = writeYamlRule.files[manifestFileName].get(DATA)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        assertAllVariablesReplaced(yamlStringAfterReplacement)
+        assertCorrectVariableResolution(manifestDataAfterReplacement)
+
+        assertDataTypeAndSubstitutionCorrectness(manifestDataAfterReplacement)
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    private void assertDataTypeAndSubstitutionCorrectness(Map<String, Object> manifestDataAfterReplacement) {
+        // See datatypes_manifest.yml and datatypes_manifest-variables.yml.
+        // Note: For debugging consider turning on YAML writing to a file in JenkinsWriteYamlRule to see the
+        // actual outcome of replacing variables (for visual inspection).
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("instances").equals(1))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("instances") instanceof Integer)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("services").get(0) instanceof String)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("booleanVariable").equals(true))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("booleanVariable") instanceof Boolean)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("floatVariable") == 0.25)
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("floatVariable") instanceof Double)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("json-variable") instanceof String)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("object-variable") instanceof Map)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("string-variable").startsWith("true-0.25-1-"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("string-variable") instanceof String)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("single-var-with-string-constants").equals("true-with-some-more-text"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("single-var-with-string-constants") instanceof String)
+    }
+
+    @Test
+    public void substituteVariables_replacesManifestIfNoOutputGiven() throws Exception {
+        // Test that makes sure that the original input manifest file is replaced with
+        // a version that has variables replaced (by deleting the original file and
+        // dumping a new one with the same name).
+
+        String manifestFileName = "test/resources/variableSubstitution/datatypes_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/datatypes_manifest-variables.yml"
+        List<String> variablesFiles = [ variablesFileName ]
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+        fileExistsRule.registerExistingFile(variablesFileName)
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Loaded manifest at ${manifestFileName}!")
+                   .expect("[CFManifestSubstituteVariables] Loaded variables file at ${variablesFileName}!")
+                   .expect("[CFManifestSubstituteVariables] Successfully deleted file '${manifestFileName}'.")
+                   .expect("[CFManifestSubstituteVariables] Replaced variables in ${manifestFileName}.")
+                   .expect("[CFManifestSubstituteVariables] Wrote output file (with variables replaced) at ${manifestFileName}.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, script: nullScript
+
+        String yamlStringAfterReplacement = writeYamlRule.files[manifestFileName].get(SERIALIZED_YAML) as String
+        Map<String, Object> manifestDataAfterReplacement = writeYamlRule.files[manifestFileName].get(DATA)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        assertAllVariablesReplaced(yamlStringAfterReplacement)
+        assertCorrectVariableResolution(manifestDataAfterReplacement)
+
+        assertDataTypeAndSubstitutionCorrectness(manifestDataAfterReplacement)
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_writesToOutputFileIfGiven() throws Exception {
+        // Test that makes sure that the output is written to the specified file and that the original input manifest
+        // file is NOT deleted / replaced.
+
+        String manifestFileName = "test/resources/variableSubstitution/datatypes_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/datatypes_manifest-variables.yml"
+        List<String> variablesFiles = [ variablesFileName ]
+        String outputFileName = "output.yml"
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+        fileExistsRule.registerExistingFile(variablesFileName)
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Loaded manifest at ${manifestFileName}!")
+            .expect("[CFManifestSubstituteVariables] Loaded variables file at ${variablesFileName}!")
+            .expect("[CFManifestSubstituteVariables] Replaced variables in ${manifestFileName}.")
+            .expect("[CFManifestSubstituteVariables] Wrote output file (with variables replaced) at ${outputFileName}.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, outputManifestFile: outputFileName,  script: nullScript
+
+        String yamlStringAfterReplacement = writeYamlRule.files[outputFileName].get(SERIALIZED_YAML) as String
+        Map<String, Object> manifestDataAfterReplacement = writeYamlRule.files[outputFileName].get(DATA)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        // make sure the input file was NOT deleted.
+        assertFalse(loggingRule.expected.contains("[CFManifestSubstituteVariables] Successfully deleted file '${manifestFileName}'."))
+
+        assertAllVariablesReplaced(yamlStringAfterReplacement)
+        assertCorrectVariableResolution(manifestDataAfterReplacement)
+
+        assertDataTypeAndSubstitutionCorrectness(manifestDataAfterReplacement)
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_resolvesConflictsProperly_InMultiYamlFiles() throws Exception {
+        // This test checks if the resolution and of conflicting variables and the
+        // overriding nature of variable lists vs. variable files lists is working properly.
+
+        String manifestFileName = "test/resources/variableSubstitution/multi_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/manifest-variables.yml"
+        String conflictingVariablesFileName = "test/resources/variableSubstitution/manifest-variables-conflicting.yml"
+        List<String> variablesFiles = [ variablesFileName, conflictingVariablesFileName ] //introducing a conflicting file whose entries should win, since it is last in the list
+
+        List<Map<String, Object>> variablesList = [
+            ["unique-prefix" : "uniquePrefix-from-vars-list"],
+            ["unique-prefix" : "uniquePrefix-from-vars-list-conflicting"] // introduce a conflict that should win, since it is last in the list.
+        ]
+
+        fileExistsRule.registerExistingFile(manifestFileName)
+        fileExistsRule.registerExistingFile(variablesFileName)
+        fileExistsRule.registerExistingFile(conflictingVariablesFileName)
+
+        // check that a proper log is written.
+        loggingRule.expect("[CFManifestSubstituteVariables] Loaded manifest at ${manifestFileName}!")
+        //.expect("[CFManifestSubstituteVariables] Loaded variables file at ${variablesFileName}!")
+            .expect("[CFManifestSubstituteVariables] Replaced variables in ${manifestFileName}.")
+            .expect("[CFManifestSubstituteVariables] Wrote output file (with variables replaced) at ${manifestFileName}.")
+
+        // execute step
+        script.step.cfManifestSubstituteVariables manifestFile: manifestFileName, manifestVariablesFiles: variablesFiles, manifestVariables: variablesList, script: nullScript
+
+        String yamlStringAfterReplacement = writeYamlRule.files[manifestFileName].get(SERIALIZED_YAML) as String
+        List<Object> manifestDataAfterReplacement = writeYamlRule.files[manifestFileName].get(DATA)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        // check that there are no unresolved variables left.
+        assertAllVariablesReplaced(yamlStringAfterReplacement)
+
+        //check that result still is a multi-YAML file.
+        assertEquals("Dumped YAML after replacement should still be a multi-YAML file.",2, manifestDataAfterReplacement.size())
+
+        // check that resolved variables have expected values
+        manifestDataAfterReplacement.each { yaml ->
+            assertCorrectVariableSubstitutionUnderConflictAndWithOverriding(yaml as Map<String, Object>)
+        }
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    private void assertCorrectVariableSubstitutionUnderConflictAndWithOverriding(Map<String, Object> manifestDataAfterReplacement) {
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("name").equals("uniquePrefix-from-vars-list-conflicting-catalog-service-odatav2-0.0.1"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("routes").get(0).get("route").equals("uniquePrefix-from-vars-list-conflicting-catalog-service-odatav2-001.cfapps.eu10.hana.ondemand.com"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("services").get(0).equals("uniquePrefix-catalog-service-odatav2-xsuaa-conflicting-from-file"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("services").get(1).equals("uniquePrefix-catalog-service-odatav2-hana"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("xsuaa-instance-name").equals("uniquePrefix-catalog-service-odatav2-xsuaa-conflicting-from-file"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("db_service_instance_name").equals("uniquePrefix-catalog-service-odatav2-hana"))
+    }
+}

test/groovy/CloudFoundryCreateServiceTest.groovy

@@ -0,0 +1,284 @@
+import com.sap.piper.JenkinsUtils
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.ExpectedException
+import org.junit.rules.RuleChain
+
+import util.BasePiperTest
+import util.JenkinsCredentialsRule
+import util.JenkinsEnvironmentRule
+import util.JenkinsDockerExecuteRule
+import util.JenkinsFileExistsRule
+import util.JenkinsLoggingRule
+import util.JenkinsReadFileRule
+import util.JenkinsShellCallRule
+import util.JenkinsStepRule
+import util.JenkinsWriteFileRule
+import util.JenkinsReadYamlRule
+import util.Rules
+
+import static org.hamcrest.Matchers.stringContainsInOrder
+import static org.junit.Assert.*
+
+import static org.hamcrest.Matchers.hasItem
+import static org.hamcrest.Matchers.is
+import static org.hamcrest.Matchers.not
+import static org.hamcrest.Matchers.hasEntry
+import static org.hamcrest.Matchers.containsString
+
+class CloudFoundryCreateServiceTest extends BasePiperTest {
+
+    private File tmpDir = File.createTempDir()
+    private ExpectedException thrown = ExpectedException.none()
+    private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this)
+    private JenkinsShellCallRule shellRule = new JenkinsShellCallRule(this)
+    private JenkinsDockerExecuteRule dockerExecuteRule = new JenkinsDockerExecuteRule(this)
+    private JenkinsStepRule stepRule = new JenkinsStepRule(this)
+    private JenkinsEnvironmentRule environmentRule = new JenkinsEnvironmentRule(this)
+    private JenkinsReadYamlRule readYamlRule = new JenkinsReadYamlRule(this)
+    private JenkinsFileExistsRule fileExistsRule = new JenkinsFileExistsRule(this)
+    private JenkinsCredentialsRule credentialsRule = new JenkinsCredentialsRule(this).withCredentials('test_cfCredentialsId', 'test_cf', '********')
+
+    private writeInfluxMap = [:]
+
+    class JenkinsUtilsMock extends JenkinsUtils {
+        def isJobStartedByUser() {
+            return true
+        }
+    }
+
+    @Rule
+    public RuleChain rules = Rules
+        .getCommonRules(this)
+        .around(readYamlRule)
+        .around(thrown)
+        .around(loggingRule)
+        .around(shellRule)
+        .around(dockerExecuteRule)
+        .around(environmentRule)
+        .around(fileExistsRule)
+        .around(credentialsRule)
+        .around(stepRule) // needs to be activated after dockerExecuteRule, otherwise executeDocker is not mocked
+
+    @Before
+    void init() {
+        helper.registerAllowedMethod('influxWriteData', [Map.class], {m ->
+            writeInfluxMap = m
+        })
+        fileExistsRule.registerExistingFile('test.yml')
+    }
+
+    @Test
+    void testVarsListNotAList() {               
+        thrown.expect(hudson.AbortException)
+        thrown.expectMessage('[cloudFoundryCreateService] ERROR: Parameter config.cloudFoundry.manifestVariables is not a List!')
+
+        stepRule.step.cloudFoundryCreateService([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',           
+            cfServiceManifest: 'test.yml',
+            cfManifestVariables: 'notAList'
+        ])       
+    }
+
+    @Test
+    void testVarsListEntryIsNotAMap() {
+        thrown.expect(hudson.AbortException)
+        thrown.expectMessage('[cloudFoundryCreateService] ERROR: Parameter config.cloudFoundry.manifestVariables.notAMap is not a Map!')
+
+        stepRule.step.cloudFoundryCreateService([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',           
+            cfServiceManifest: 'test.yml',
+            cfManifestVariables: ['notAMap']
+        ])       
+    }
+
+    @Test
+    void testVarsFilesListIsNotAList() {
+        thrown.expect(hudson.AbortException)
+        thrown.expectMessage('[cloudFoundryCreateService] ERROR: Parameter config.cloudFoundry.manifestVariablesFiles is not a List!')
+
+        stepRule.step.cloudFoundryCreateService([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',           
+            cfServiceManifest: 'test.yml',
+            cfManifestVariablesFiles: 'notAList'
+        ])       
+    }
+
+    @Test
+    void testRunCreateServicePushPlugin() {
+        stepRule.step.cloudFoundryCreateService([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',           
+            cfServiceManifest: 'test.yml'
+        ]) 
+
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'ppiper/cf-cli'))
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper'))        
+        assertThat(shellRule.shell, hasItem(containsString("cf login -u 'test_cf' -p '********' -a https://api.cf.eu10.hana.ondemand.com -o 'testOrg' -s 'testSpace'")))
+        assertThat(shellRule.shell, hasItem(containsString(" cf create-service-push --no-push -f 'test.yml'")))
+        assertThat(shellRule.shell, hasItem(containsString("cf logout")))
+    }
+
+    @Test
+    void testWithVariableSubstitutionFromVarsListAndVarsFile() {
+        String varsFileName='vars.yml'
+        fileExistsRule.registerExistingFile(varsFileName)
+        List varsList = [["appName" : "testApplicationFromVarsList"]]
+
+        stepRule.step.cloudFoundryCreateService([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',           
+            cfServiceManifest: 'test.yml',
+            cfManifestVariablesFiles: [varsFileName],
+            cfManifestVariables: varsList
+        ]) 
+
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'ppiper/cf-cli'))
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper'))        
+        assertThat(shellRule.shell, hasItem(containsString("cf login -u 'test_cf' -p '********' -a https://api.cf.eu10.hana.ondemand.com -o 'testOrg' -s 'testSpace'")))
+        assertThat(shellRule.shell, hasItem(containsString("cf create-service-push --no-push -f 'test.yml' --var appName='testApplicationFromVarsList' --vars-file 'vars.yml'")))
+        assertThat(shellRule.shell, hasItem(containsString("cf logout")))
+    }
+
+    @Test
+    void testEscapesUsernameAndPasswordInShellCall() {
+        credentialsRule.credentials.put('escape_cfCredentialsId',[user:"aUserWithA'",passwd:"passHasA'"])
+
+        stepRule.step.cloudFoundryCreateService([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'escape_cfCredentialsId',           
+            cfServiceManifest: 'test.yml'
+        ]) 
+
+        assertThat(shellRule.shell, hasItem(containsString("""cf login -u 'aUserWithA'"'"'' -p 'passHasA'"'"'' -a https://api.cf.eu10.hana.ondemand.com -o 'testOrg' -s 'testSpace'""")))
+    }
+
+    @Test
+    void testEscapesSpaceNameInShellCall() {
+        stepRule.step.cloudFoundryCreateService([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: "testSpaceWith'",
+            cfCredentialsId: 'test_cfCredentialsId',           
+            cfServiceManifest: 'test.yml'
+        ]) 
+        assertThat(shellRule.shell, hasItem(containsString("""cf login -u 'test_cf' -p '********' -a https://api.cf.eu10.hana.ondemand.com -o 'testOrg' -s 'testSpaceWith'"'"''""")))
+    }
+
+    @Test
+    void testEscapesOrgNameInShellCall() {
+        stepRule.step.cloudFoundryCreateService([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: "testOrgWith'",
+            cfSpace: "testSpace",
+            cfCredentialsId: 'test_cfCredentialsId',           
+            cfServiceManifest: 'test.yml'
+        ]) 
+        assertThat(shellRule.shell, hasItem(containsString("""cf login -u 'test_cf' -p '********' -a https://api.cf.eu10.hana.ondemand.com -o 'testOrgWith'"'"'' -s 'testSpace'""")))
+    }
+
+    @Test
+    void testWithVariableSubstitutionFromVarsListGetsEscaped() {
+        List varsList = [["appName" : "testApplicationFromVarsListWith'"]]
+
+        stepRule.step.cloudFoundryCreateService([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',           
+            cfServiceManifest: 'test.yml',
+            cfManifestVariables: varsList
+        ]) 
+
+        assertThat(shellRule.shell, hasItem(containsString("""cf create-service-push --no-push -f 'test.yml' --var appName='testApplicationFromVarsListWith'"'"''""")))
+    }
+
+    @Test
+    void testWithVariableSubstitutionFromVarsFilesGetsEscaped() {
+        String varsFileName="varsWith'.yml"
+        fileExistsRule.registerExistingFile(varsFileName)
+
+        stepRule.step.cloudFoundryCreateService([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',           
+            cfServiceManifest: 'test.yml',
+            cfManifestVariablesFiles: [varsFileName]
+        ]) 
+
+        assertThat(shellRule.shell, hasItem(containsString("""cf create-service-push --no-push -f 'test.yml' --vars-file 'varsWith'"'"'.yml'""")))
+    }
+
+    @Test
+    void testCfLogoutHappensEvenWhenCreateServiceFails() {
+
+        thrown.expect(hudson.AbortException)
+        thrown.expectMessage('[cloudFoundryCreateService] ERROR: The execution of the create-service-push plugin failed, see the logs above for more details.')
+
+        shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX,/(create-service-push)/,128)
+
+        stepRule.step.cloudFoundryCreateService([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',           
+            cfServiceManifest: 'test.yml'
+        ]) 
+
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'ppiper/cf-cli'))
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper'))        
+        assertThat(shellRule.shell, hasItem(containsString("cf login -u 'test_cf' -p '********' -a https://api.cf.eu10.hana.ondemand.com -o 'testOrg' -s 'testSpace'")))
+        assertThat(shellRule.shell, hasItem(containsString(" cf create-service-push --no-push -f 'test.yml'")))
+        assertThat(shellRule.shell, hasItem(containsString("cf logout")))
+    } 
+}

test/groovy/CloudFoundryDeployTest.groovy

@@ -9,6 +9,7 @@ import util.BasePiperTest
 import util.JenkinsCredentialsRule
 import util.JenkinsEnvironmentRule
 import util.JenkinsDockerExecuteRule
+import util.JenkinsFileExistsRule
 import util.JenkinsLoggingRule
 import util.JenkinsReadFileRule
 import util.JenkinsShellCallRule
@@ -18,12 +19,13 @@ import util.JenkinsReadYamlRule
 import util.Rules
 
 import static org.hamcrest.Matchers.stringContainsInOrder
-import static org.junit.Assert.assertThat
+import static org.junit.Assert.*
 
 import static org.hamcrest.Matchers.hasItem
 import static org.hamcrest.Matchers.is
 import static org.hamcrest.Matchers.not
 import static org.hamcrest.Matchers.hasEntry
+import static org.hamcrest.Matchers.allOf
 import static org.hamcrest.Matchers.containsString
 
 class CloudFoundryDeployTest extends BasePiperTest {
@@ -38,6 +40,7 @@ class CloudFoundryDeployTest extends BasePiperTest {
     private JenkinsStepRule stepRule = new JenkinsStepRule(this)
     private JenkinsEnvironmentRule environmentRule = new JenkinsEnvironmentRule(this)
     private JenkinsReadYamlRule readYamlRule = new JenkinsReadYamlRule(this)
+    private JenkinsFileExistsRule fileExistsRule = new JenkinsFileExistsRule(this, [])
 
     private writeInfluxMap = [:]
 
@@ -56,6 +59,7 @@ class CloudFoundryDeployTest extends BasePiperTest {
         .around(shellRule)
         .around(writeFileRule)
         .around(readFileRule)
+        .around(fileExistsRule)
         .around(dockerExecuteRule)
         .around(environmentRule)
         .around(new JenkinsCredentialsRule(this).withCredentials('test_cfCredentialsId', 'test_cf', '********'))
@@ -96,6 +100,7 @@ class CloudFoundryDeployTest extends BasePiperTest {
         ])
         // asserts
         assertThat(loggingRule.log, containsString('[cloudFoundryDeploy] General parameters: deployTool=, deployType=standard, cfApiEndpoint=https://api.cf.eu10.hana.ondemand.com, cfOrg=testOrg, cfSpace=testSpace, cfCredentialsId=myCreds'))
+        assertThat(loggingRule.log, containsString('[cloudFoundryDeploy] WARNING! Found unsupported deployTool. Skipping deployment.'))
     }
 
     @Test
@@ -125,6 +130,7 @@ class CloudFoundryDeployTest extends BasePiperTest {
         ])
         // asserts
         assertThat(loggingRule.log, containsString('[cloudFoundryDeploy] General parameters: deployTool=notAvailable, deployType=standard, cfApiEndpoint=https://api.cf.eu10.hana.ondemand.com, cfOrg=testOrg, cfSpace=testSpace, cfCredentialsId=myCreds'))
+        assertThat(loggingRule.log, containsString('[cloudFoundryDeploy] WARNING! Found unsupported deployTool. Skipping deployment.'))
     }
 
     @Test
@@ -208,7 +214,7 @@ class CloudFoundryDeployTest extends BasePiperTest {
 
     @Test
     void testCfNativeAppNameFromManifest() {
-        helper.registerAllowedMethod('fileExists', [String.class], { s -> return true })
+        fileExistsRule.registerExistingFile('test.yml')
         readYamlRule.registerYaml('test.yml', "applications: [[name: 'manifestAppName']]")
         helper.registerAllowedMethod('writeYaml', [Map], { Map parameters ->
             generatedFile = parameters.file
@@ -233,7 +239,7 @@ class CloudFoundryDeployTest extends BasePiperTest {
 
     @Test
     void testCfNativeWithoutAppName() {
-        helper.registerAllowedMethod('fileExists', [String.class], { s -> return true })
+        fileExistsRule.registerExistingFile('test.yml')
         readYamlRule.registerYaml('test.yml', "applications: [[]]")
         helper.registerAllowedMethod('writeYaml', [Map], { Map parameters ->
             generatedFile = parameters.file
@@ -351,7 +357,7 @@ class CloudFoundryDeployTest extends BasePiperTest {
         readYamlRule.registerYaml('test.yml', "applications: [[]]")
 
         thrown.expect(hudson.AbortException)
-        thrown.expectMessage("Could not stop application testAppName-old. Error: any error message")
+        thrown.expectMessage("[cloudFoundryDeploy] ERROR: Could not stop application testAppName-old. Error: any error message")
 
         stepRule.step.cloudFoundryDeploy([
             script: nullScript,
@@ -400,7 +406,7 @@ class CloudFoundryDeployTest extends BasePiperTest {
     @Test
     void testCfNativeWithoutAppNameBlueGreen() {
 
-        helper.registerAllowedMethod('fileExists', [String.class], { s -> return true })
+        fileExistsRule.registerExistingFile('test.yml')
         readYamlRule.registerYaml('test.yml', "applications: [[]]")
 
         thrown.expect(hudson.AbortException)
@@ -419,6 +425,36 @@ class CloudFoundryDeployTest extends BasePiperTest {
         ])
     }
 
+    @Test
+    void testCfNativeFailureInShellCall() {
+        readYamlRule.registerYaml('test.yml', "applications: [[name: 'manifestAppName']]")
+        helper.registerAllowedMethod('writeYaml', [Map], { Map parameters ->
+            generatedFile = parameters.file
+            data = parameters.data
+        })
+        shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX,/(cf login -u "test_cf")/,1)
+
+        thrown.expect(hudson.AbortException)
+        thrown.expectMessage('[cloudFoundryDeploy] ERROR: The execution of the deploy command failed, see the log for details.')
+
+
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',
+            cfAppName: 'testAppName',
+            cfManifest: 'test.yml'
+        ])
+
+        assertThat(shellRule.shell, hasItem(containsString('cf login -u "test_cf" -p \'********\' -a https://api.cf.eu10.hana.ondemand.com -o "testOrg" -s "testSpace"')))
+        assertThat(shellRule.shell, hasItem(containsString("cf push testAppName -f 'test.yml'")))
+        assertThat(shellRule.shell, hasItem(containsString("cf logout")))
+    }
+
 
     @Test
     void testMta() {
@@ -435,7 +471,7 @@ class CloudFoundryDeployTest extends BasePiperTest {
         // asserts
         assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 's4sdk/docker-cf-cli'))
         assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper'))
-        assertThat(shellRule.shell, hasItem(containsString('cf login -u test_cf -p \'********\' -a https://api.cf.eu10.hana.ondemand.com -o "testOrg" -s "testSpace"')))
+        assertThat(shellRule.shell, hasItem(containsString('cf login -u "test_cf" -p \'********\' -a https://api.cf.eu10.hana.ondemand.com -o "testOrg" -s "testSpace"')))
         assertThat(shellRule.shell, hasItem(containsString('cf deploy target/test.mtar -f')))
         assertThat(shellRule.shell, hasItem(containsString('cf logout')))
     }
@@ -455,7 +491,7 @@ class CloudFoundryDeployTest extends BasePiperTest {
             mtaPath: 'target/test.mtar'
         ])
 
-        assertThat(shellRule.shell, hasItem(stringContainsInOrder(["cf login -u test_cf", 'cf bg-deploy', '-f', '--no-confirm'])))
+        assertThat(shellRule.shell, hasItem(stringContainsInOrder(["cf login -u \"test_cf\"", 'cf bg-deploy', '-f', '--no-confirm'])))
     }
 
     @Test
@@ -490,4 +526,364 @@ class CloudFoundryDeployTest extends BasePiperTest {
         assertThat(writeInfluxMap.customDataMapTags.deployment_data.cfSpace, is('testSpace'))
     }
 
+    @Test
+    void testCfPushDeploymentWithVariableSubstitutionFromFile() {
+        readYamlRule.registerYaml('test.yml', "applications: [[name: '((appName))']]")
+        fileExistsRule.registerExistingFile('test.yml')
+        fileExistsRule.registerExistingFile('vars.yml')
+
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',
+            cfAppName: 'testAppName',
+            cfManifest: 'test.yml',
+            cfManifestVariablesFiles: ['vars.yml']
+        ])
+
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 's4sdk/docker-cf-cli'))
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper'))
+        assertThat(dockerExecuteRule.dockerParams.dockerEnvVars, hasEntry('STATUS_CODE', "${200}"))
+        assertThat(shellRule.shell, hasItem(containsString('cf login -u "test_cf" -p \'********\' -a https://api.cf.eu10.hana.ondemand.com -o "testOrg" -s "testSpace"')))
+        assertThat(shellRule.shell, hasItem(containsString("cf push testAppName --vars-file 'vars.yml' -f 'test.yml'")))
+        assertThat(shellRule.shell, hasItem(containsString("cf logout")))
+        assertThat(loggingRule.log,containsString("We will add the following string to the cf push call: --vars-file 'vars.yml' !"))
+        assertThat(loggingRule.log,not(containsString("We will add the following string to the cf push call:  !")))
+    }
+
+    @Test
+    void testCfPushDeploymentWithVariableSubstitutionFromNotExistingFilePrintsWarning() {
+        readYamlRule.registerYaml('test.yml', "applications: [[name: '((appName))']]")
+        fileExistsRule.registerExistingFile('test.yml')        
+
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',
+            cfAppName: 'testAppName',
+            cfManifest: 'test.yml',
+            cfManifestVariablesFiles: ['vars.yml']
+        ])
+
+        // asserts
+        assertThat(shellRule.shell, hasItem(containsString("cf push testAppName -f 'test.yml'")))       
+        assertThat(loggingRule.log, containsString("[WARNING] We skip adding not-existing file 'vars.yml' as a vars-file to the cf create-service-push call"))
+    }
+
+    @Test
+    void testCfPushDeploymentWithVariableSubstitutionFromVarsList() {
+        readYamlRule.registerYaml('test.yml', "applications: [[name: '((appName))']]")
+        List varsList = [["appName" : "testApplicationFromVarsList"]]                
+        
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',
+            cfAppName: 'testAppName',
+            cfManifest: 'test.yml',
+            cfManifestVariables: varsList
+        ])
+
+        // asserts
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 's4sdk/docker-cf-cli'))
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper'))
+        assertThat(dockerExecuteRule.dockerParams.dockerEnvVars, hasEntry('STATUS_CODE', "${200}"))
+        assertThat(shellRule.shell, hasItem(containsString('cf login -u "test_cf" -p \'********\' -a https://api.cf.eu10.hana.ondemand.com -o "testOrg" -s "testSpace"')))
+        assertThat(shellRule.shell, hasItem(containsString("cf push testAppName --var appName='testApplicationFromVarsList' -f 'test.yml'")))
+        assertThat(shellRule.shell, hasItem(containsString("cf logout")))
+        assertThat(loggingRule.log,containsString("We will add the following string to the cf push call: --var appName='testApplicationFromVarsList' !"))
+        assertThat(loggingRule.log,not(containsString("We will add the following string to the cf push call:  !")))
+    }
+
+    @Test
+    void testCfPushDeploymentWithVariableSubstitutionFromVarsListNotAList() {
+        readYamlRule.registerYaml('test.yml', "applications: [[name: '((appName))']]")        
+        
+        thrown.expect(hudson.AbortException)
+        thrown.expectMessage('[cloudFoundryDeploy] ERROR: Parameter config.cloudFoundry.manifestVariables is not a List!')
+
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',
+            cfAppName: 'testAppName',
+            cfManifest: 'test.yml',
+            cfManifestVariables: 'notAList'
+        ])
+       
+    }
+
+    @Test
+    void testCfPushDeploymentWithVariableSubstitutionFromVarsListAndVarsFile() {
+        readYamlRule.registerYaml('test.yml', "applications: [[name: '((appName))']]")
+        List varsList = [["appName" : "testApplicationFromVarsList"]]
+        fileExistsRule.registerExistingFile('vars.yml')
+
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',
+            cfAppName: 'testAppName',
+            cfManifest: 'test.yml',
+            cfManifestVariablesFiles: ['vars.yml'],
+            cfManifestVariables: varsList
+        ])
+
+        // asserts
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 's4sdk/docker-cf-cli'))
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper'))
+        assertThat(dockerExecuteRule.dockerParams.dockerEnvVars, hasEntry('STATUS_CODE', "${200}"))
+        assertThat(shellRule.shell, hasItem(containsString('cf login -u "test_cf" -p \'********\' -a https://api.cf.eu10.hana.ondemand.com -o "testOrg" -s "testSpace"')))
+        assertThat(shellRule.shell, hasItem(containsString("cf push testAppName --var appName='testApplicationFromVarsList' --vars-file 'vars.yml' -f 'test.yml'")))
+        assertThat(shellRule.shell, hasItem(containsString("cf logout")))
+    }
+
+    @Test
+    void testCfPushDeploymentWithoutVariableSubstitution() {
+        readYamlRule.registerYaml('test.yml', "applications: [[name: '((appName))']]")
+
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',
+            cfAppName: 'testAppName',
+            cfManifest: 'test.yml'
+        ])
+
+        // asserts        
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 's4sdk/docker-cf-cli'))
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper'))
+        assertThat(dockerExecuteRule.dockerParams.dockerEnvVars, hasEntry('STATUS_CODE', "${200}"))
+        assertThat(shellRule.shell, hasItem(containsString('cf login -u "test_cf" -p \'********\' -a https://api.cf.eu10.hana.ondemand.com -o "testOrg" -s "testSpace"')))
+        assertThat(shellRule.shell, hasItem(containsString("cf push testAppName -f 'test.yml'")))
+        assertThat(shellRule.shell, hasItem(containsString("cf logout")))
+    }
+
+    @Test
+    void testCfBlueGreenDeploymentWithVariableSubstitution() {
+
+        readYamlRule.registerYaml('test.yml', "applications: [[name: '((appName))']]")
+        readYamlRule.registerYaml('vars.yml', "[appName: 'testApplication']")
+
+        fileExistsRule.registerExistingFile("test.yml")
+        fileExistsRule.registerExistingFile("vars.yml")
+
+        boolean testYamlWritten = false
+        def testYamlData = null
+        helper.registerAllowedMethod('writeYaml', [Map], { Map m ->
+            if (m.file.equals("test.yml")) {
+                testYamlWritten = true
+                testYamlData = m.data
+            }
+        })
+
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            deployType: 'blue-green',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',
+            cfAppName: 'testAppName',
+            cfManifest: 'test.yml',
+            cfManifestVariablesFiles: ['vars.yml']
+        ])
+
+        // asserts
+        assertTrue(testYamlWritten)
+        assertNotNull(testYamlData)
+        assertThat(testYamlData.get("applications").get(0).get(0).get("name"), is("testApplication"))
+
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 's4sdk/docker-cf-cli'))
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper'))
+        assertThat(dockerExecuteRule.dockerParams.dockerEnvVars, hasEntry('STATUS_CODE', "${200}"))
+        assertThat(shellRule.shell, hasItem(containsString('cf login -u "test_cf" -p \'********\' -a https://api.cf.eu10.hana.ondemand.com -o "testOrg" -s "testSpace"')))
+        assertThat(shellRule.shell, hasItem(containsString("cf blue-green-deploy testAppName --delete-old-apps -f 'test.yml'")))
+        assertThat(shellRule.shell, hasItem(containsString("cf logout")))
+    }
+
+    @Test
+    void testCfBlueGreenDeploymentWithVariableSubstitutionFromVarsList() {
+        readYamlRule.registerYaml('test.yml', "applications: [[name: '((appName))']]")
+        readYamlRule.registerYaml('vars.yml', "[appName: 'testApplication']")
+        List varsList = [["appName" : "testApplicationFromVarsList"]]
+
+        fileExistsRule.registerExistingFile("test.yml")
+        fileExistsRule.registerExistingFile("vars.yml")
+
+        boolean testYamlWritten = false
+        def testYamlData = null
+        helper.registerAllowedMethod('writeYaml', [Map], { Map m ->
+            if (m.file.equals("test.yml")) {
+                testYamlWritten = true
+                testYamlData = m.data
+            }
+        })
+
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            deployType: 'blue-green',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            cfCredentialsId: 'test_cfCredentialsId',
+            cfAppName: 'testAppName',
+            cfManifest: 'test.yml',
+            cfManifestVariablesFiles: ['vars.yml'],
+            cfManifestVariables: varsList
+        ])
+
+        // asserts
+        assertTrue(testYamlWritten)
+        assertNotNull(testYamlData)
+        assertThat(testYamlData.get("applications").get(0).get(0).get("name"), is("testApplicationFromVarsList"))
+
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 's4sdk/docker-cf-cli'))
+        assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper'))
+        assertThat(dockerExecuteRule.dockerParams.dockerEnvVars, hasEntry('STATUS_CODE', "${200}"))
+        assertThat(shellRule.shell, hasItem(containsString('cf login -u "test_cf" -p \'********\' -a https://api.cf.eu10.hana.ondemand.com -o "testOrg" -s "testSpace"')))
+        assertThat(shellRule.shell, hasItem(containsString("cf blue-green-deploy testAppName --delete-old-apps -f 'test.yml'")))
+        assertThat(shellRule.shell, hasItem(containsString("cf logout")))
+    }
+
+    @Test
+    void testTraceOutputOnVerbose() {
+
+        fileExistsRule.existingFiles.addAll(
+            'test.yml',
+            'cf.log'
+        )
+
+        new File(tmpDir, 'cf.log') << 'Hello SAP'
+
+        readYamlRule.registerYaml('test.yml', "applications: [[name: 'manifestAppName']]")
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            cloudFoundry: [
+                org: 'testOrg',
+                space: 'testSpace',
+                manifest: 'test.yml',
+                ],
+            cfCredentialsId: 'test_cfCredentialsId',
+            verbose: true
+        ])
+
+        assertThat(loggingRule.log, allOf(
+            containsString('### START OF CF CLI TRACE OUTPUT ###'),
+            containsString('Hello SAP'),
+            containsString('### END OF CF CLI TRACE OUTPUT ###')))
+    }
+
+    @Test
+    void testTraceNoTraceFileWritten() {
+
+        fileExistsRule.existingFiles.addAll(
+            'test.yml',
+        )
+
+        readYamlRule.registerYaml('test.yml', "applications: [[name: 'manifestAppName']]")
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            cloudFoundry: [
+                org: 'testOrg',
+                space: 'testSpace',
+                manifest: 'test.yml',
+                ],
+            cfCredentialsId: 'test_cfCredentialsId',
+            verbose: true
+        ])
+
+        assertThat(loggingRule.log, containsString('No trace file found'))
+    }
+
+    @Test
+    void testAdditionCfNativeOpts() {
+
+        readYamlRule.registerYaml('test.yml', "applications: [[name: 'manifestAppName']]")
+        helper.registerAllowedMethod('writeYaml', [Map], { Map parameters ->
+            generatedFile = parameters.file
+            data = parameters.data
+        })
+        nullScript.commonPipelineEnvironment.setArtifactVersion('1.2.3')
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            deployTool: 'cf_native',
+            cfOrg: 'testOrg',
+            cfSpace: 'testSpace',
+            loginParameters: '--some-login-opt value',
+            cfNativeDeployParameters: '--some-deploy-opt cf-value',
+            cfCredentialsId: 'test_cfCredentialsId',
+            cfAppName: 'testAppName',
+            cfManifest: 'test.yml'
+        ])
+
+        assertThat(shellRule.shell, hasItem(
+            stringContainsInOrder([
+                'cf login ', '--some-login-opt value',
+                'cf push', '--some-deploy-opt cf-value'])))
+
+    }
+
+    @Test
+    void testAdditionMtaOpts() {
+
+        stepRule.step.cloudFoundryDeploy([
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: new JenkinsUtilsMock(),
+            cloudFoundry: [
+                org: 'testOrg',
+                space: 'testSpace',
+            ],
+            apiParameters: '--some-api-opt value',
+            loginParameters: '--some-login-opt value',
+            mtaDeployParameters: '--some-deploy-opt mta-value',
+            cfCredentialsId: 'test_cfCredentialsId',
+            deployTool: 'mtaDeployPlugin',
+            deployType: 'blue-green',
+            mtaPath: 'target/test.mtar'
+        ])
+
+        assertThat(shellRule.shell, hasItem(
+            stringContainsInOrder([
+                'cf api', '--some-api-opt value',
+                'cf login ', '--some-login-opt value',
+                'cf bg-deploy', '--some-deploy-opt mta-value'])))
+
+    }
+
 }

test/groovy/DockerExecuteOnKubernetesTest.groovy

@@ -58,7 +58,7 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
     def pullImageMap = [:]
     def namespace
     def securityContext
-    Map stashMap
+    List stashList = []
 
     @Before
     void init() {
@@ -96,7 +96,7 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
             body()
         })
         helper.registerAllowedMethod('stash', [Map.class], {m ->
-            stashMap = m
+            stashList.add(m)
         })
 
     }
@@ -228,7 +228,7 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
     }
 
     @Test
-    void testSidecarDefault() {
+    void testSidecarDefaultWithContainerMap() {
         List portMapping = []
         helper.registerAllowedMethod('portMapping', [Map.class], {m ->
             portMapping.add(m)
@@ -273,6 +273,36 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
         assertThat(envList, hasItem(hasItem(allOf(hasEntry('name', 'customEnvKey'), hasEntry ('value','customEnvValue')))))
     }
 
+    @Test
+    void testSidecarDefaultWithParameters() {
+        List portMapping = []
+        helper.registerAllowedMethod('portMapping', [Map.class], {m ->
+            portMapping.add(m)
+            return m
+        })
+        stepRule.step.dockerExecuteOnKubernetes(
+            script: nullScript,
+            juStabUtils: utils,
+            containerMap: ['maven:3.5-jdk-8-alpine': 'mavenexecute'],
+            containerName: 'mavenexecute',
+            dockerOptions: '-it',
+            dockerVolumeBind: ['my_vol': '/my_vol'],
+            dockerEnvVars: ['http_proxy': 'http://proxy:8000'],
+            dockerWorkspace: '/home/piper',
+            sidecarEnvVars: ['testEnv': 'testVal'],
+            sidecarImage: 'postgres',
+            sidecarName: 'postgres',
+            sidecarReadyCommand: 'pg_isready'
+        ) {
+            bodyExecuted = true
+        }
+
+        assertThat(bodyExecuted, is(true))
+
+        assertThat(containersList, allOf(hasItem('postgres'), hasItem('mavenexecute')))
+        assertThat(imageList, allOf(hasItem('maven:3.5-jdk-8-alpine'), hasItem('postgres')))
+    }
+
     @Test
     void testDockerExecuteOnKubernetesWithCustomShell() {
         stepRule.step.dockerExecuteOnKubernetes(
@@ -389,7 +419,7 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
     @Test
     void testDockerExecuteOnKubernetesCustomJnlpViaEnv() {
 
-        nullScript.configuration = [
+        nullScript.commonPipelineEnvironment.configuration = [
             general: [jenkinsKubernetes: [jnlpAgent: 'config/jnlp:latest']]
         ]
         binding.variables.env.JENKINS_JNLP_IMAGE = 'env/jnlp:latest'
@@ -413,10 +443,10 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
     @Test
     void testDockerExecuteOnKubernetesCustomJnlpViaConfig() {
 
-        nullScript.configuration = [
+        nullScript.commonPipelineEnvironment.configuration = [
             general: [jenkinsKubernetes: [jnlpAgent: 'config/jnlp:latest']]
         ]
-        binding.variables.env.JENKINS_JNLP_IMAGE = 'config/jnlp:latest'
+        //binding.variables.env.JENKINS_JNLP_IMAGE = 'config/jnlp:latest'
         stepRule.step.dockerExecuteOnKubernetes(
             script: nullScript,
             juStabUtils: utils,
@@ -434,6 +464,33 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
         ))
     }
 
+    @Test
+    void tastStashIncludesAndExcludes() {
+        nullScript.commonPipelineEnvironment.configuration = [
+            steps: [
+                dockerExecuteOnKubernetes: [
+                    stashExcludes: [
+                        workspace: 'workspace/exclude.test',
+                        stashBack: 'container/exclude.test'
+                    ],
+                    stashIncludes: [
+                        workspace: 'workspace/include.test',
+                        stashBack: 'container/include.test'
+                    ]
+                ]
+            ]
+        ]
+        stepRule.step.dockerExecuteOnKubernetes(
+            script: nullScript,
+            juStabUtils: utils,
+            dockerImage: 'maven:3.5-jdk-8-alpine',
+        ) {
+            bodyExecuted = true
+        }
+        assertThat(stashList[0], allOf(hasEntry('includes','workspace/include.test'), hasEntry('excludes','workspace/exclude.test')))
+        assertThat(stashList[1], allOf(hasEntry('includes','container/include.test'), hasEntry('excludes','container/exclude.test')))
+    }
+
 
     private container(options, body) {
         containerName = options.name

test/groovy/DockerExecuteTest.groovy

@@ -1,6 +1,6 @@
 import com.sap.piper.k8s.ContainerMap
 import com.sap.piper.JenkinsUtils
-
+import com.sap.piper.SidecarUtils
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
@@ -41,17 +41,22 @@ class DockerExecuteTest extends BasePiperTest {
     void init() {
         bodyExecuted = false
         docker = new DockerMock()
-        JenkinsUtils.metaClass.static.isPluginActive = {def s -> new PluginMock(s).isActive()}
+        JenkinsUtils.metaClass.static.isPluginActive = { def s -> new PluginMock(s).isActive() }
         binding.setVariable('docker', docker)
         shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "docker .*", 0)
     }
 
     @Test
     void testExecuteInsideContainerOfExistingPod() throws Exception {
+        List usedDockerEnvVars
         helper.registerAllowedMethod('container', [String.class, Closure.class], { String container, Closure body ->
             containerName = container
             body()
         })
+        helper.registerAllowedMethod('withEnv', [List.class, Closure.class], { List envVars, Closure body ->
+            usedDockerEnvVars = envVars
+            body()
+        })
         binding.setVariable('env', [POD_NAME: 'testpod', ON_K8S: 'true'])
         ContainerMap.instance.setMap(['testpod': ['maven:3.5-jdk-8-alpine': 'mavenexec']])
         stepRule.step.dockerExecute(script: nullScript,
@@ -61,8 +66,9 @@ class DockerExecuteTest extends BasePiperTest {
         }
         assertTrue(loggingRule.log.contains('Executing inside a Kubernetes Container'))
         assertEquals('mavenexec', containerName)
+        assertEquals(usedDockerEnvVars[0].toString(), "http_proxy=http://proxy:8000")
         assertTrue(bodyExecuted)
-     }
+    }
 
     @Test
     void testExecuteInsideNewlyCreatedPod() throws Exception {
@@ -96,7 +102,7 @@ class DockerExecuteTest extends BasePiperTest {
     void testExecuteInsidePodWithStageKeyEmptyValue() throws Exception {
         helper.registerAllowedMethod('dockerExecuteOnKubernetes', [Map.class, Closure.class], { Map config, Closure body -> body() })
         binding.setVariable('env', [POD_NAME: 'testpod', ON_K8S: 'true'])
-        ContainerMap.instance.setMap(['testpod':[:]])
+        ContainerMap.instance.setMap(['testpod': [:]])
         stepRule.step.dockerExecute(script: nullScript,
             dockerImage: 'maven:3.5-jdk-8-alpine',
             dockerEnvVars: ['http_proxy': 'http://proxy:8000']) {
@@ -109,7 +115,7 @@ class DockerExecuteTest extends BasePiperTest {
     @Test
     void testExecuteInsidePodWithCustomCommandAndShell() throws Exception {
         Map kubernetesConfig = [:]
-        helper.registerAllowedMethod('dockerExecuteOnKubernetes', [Map.class, Closure.class], {Map config, Closure body ->
+        helper.registerAllowedMethod('dockerExecuteOnKubernetes', [Map.class, Closure.class], { Map config, Closure body ->
             kubernetesConfig = config
             return body()
         })
@@ -119,7 +125,7 @@ class DockerExecuteTest extends BasePiperTest {
             containerCommand: '/busybox/tail -f /dev/null',
             containerShell: '/busybox/sh',
             dockerImage: 'maven:3.5-jdk-8-alpine'
-        ){
+        ) {
             bodyExecuted = true
         }
         assertTrue(loggingRule.log.contains('Executing inside a Kubernetes Pod'))
@@ -141,7 +147,7 @@ class DockerExecuteTest extends BasePiperTest {
 
     @Test
     void testSkipDockerImagePull() throws Exception {
-        nullScript.commonPipelineEnvironment.configuration = [steps:[dockerExecute:[dockerPullImage: false]]]
+        nullScript.commonPipelineEnvironment.configuration = [steps: [dockerExecute: [dockerPullImage: false]]]
         stepRule.step.dockerExecute(
             script: nullScript,
             dockerImage: 'maven:3.5-jdk-8-alpine'
@@ -158,11 +164,11 @@ class DockerExecuteTest extends BasePiperTest {
             script: nullScript,
             dockerName: 'maven',
             dockerImage: 'maven:3.5-jdk-8-alpine',
-            sidecarEnvVars: ['testEnv':'testVal'],
+            sidecarEnvVars: ['testEnv': 'testVal'],
             sidecarImage: 'selenium/standalone-chrome',
-            sidecarVolumeBind: ['/dev/shm':'/dev/shm'],
+            sidecarVolumeBind: ['/dev/shm': '/dev/shm'],
             sidecarName: 'testAlias',
-            sidecarPorts: ['4444':'4444', '1111':'1111'],
+            sidecarPorts: ['4444': '4444', '1111': '1111'],
             sidecarPullImage: false
         ) {
             bodyExecuted = true
@@ -174,10 +180,10 @@ class DockerExecuteTest extends BasePiperTest {
     @Test
     void testExecuteInsideDockerContainerWithParameters() throws Exception {
         stepRule.step.dockerExecute(script: nullScript,
-                      dockerImage: 'maven:3.5-jdk-8-alpine',
-                      dockerOptions: '-description=lorem ipsum',
-                      dockerVolumeBind: ['my_vol': '/my_vol'],
-                      dockerEnvVars: ['http_proxy': 'http://proxy:8000']) {
+            dockerImage: 'maven:3.5-jdk-8-alpine',
+            dockerOptions: '-description=lorem ipsum',
+            dockerVolumeBind: ['my_vol': '/my_vol'],
+            dockerEnvVars: ['http_proxy': 'http://proxy:8000']) {
             bodyExecuted = true
         }
         assertTrue(docker.getParameters().contains('--env https_proxy '))
@@ -215,16 +221,16 @@ class DockerExecuteTest extends BasePiperTest {
     }
 
     @Test
-    void testSidecarDefault(){
+    void testSidecarDefault() {
         stepRule.step.dockerExecute(
             script: nullScript,
             dockerName: 'maven',
             dockerImage: 'maven:3.5-jdk-8-alpine',
-            sidecarEnvVars: ['testEnv':'testVal'],
+            sidecarEnvVars: ['testEnv': 'testVal'],
             sidecarImage: 'selenium/standalone-chrome',
-            sidecarVolumeBind: ['/dev/shm':'/dev/shm'],
+            sidecarVolumeBind: ['/dev/shm': '/dev/shm'],
             sidecarName: 'testAlias',
-            sidecarPorts: ['4444':'4444', '1111':'1111']
+            sidecarPorts: ['4444': '4444', '1111': '1111']
         ) {
             bodyExecuted = true
         }
@@ -244,7 +250,7 @@ class DockerExecuteTest extends BasePiperTest {
     }
 
     @Test
-    void testSidecarHealthCheck(){
+    void testSidecarHealthCheck() {
         stepRule.step.dockerExecute(
             script: nullScript,
             dockerImage: 'maven:3.5-jdk-8-alpine',
@@ -256,18 +262,19 @@ class DockerExecuteTest extends BasePiperTest {
     }
 
     @Test
-    void testSidecarKubernetes(){
+    void testSidecarKubernetes() {
         boolean dockerExecuteOnKubernetesCalled = false
         binding.setVariable('env', [ON_K8S: 'true'])
         helper.registerAllowedMethod('dockerExecuteOnKubernetes', [Map.class, Closure.class], { params, body ->
             dockerExecuteOnKubernetesCalled = true
-            assertThat(params.containerCommands['selenium/standalone-chrome'], is(''))
-            assertThat(params.containerEnvVars, allOf(hasEntry('selenium/standalone-chrome', ['testEnv': 'testVal']),hasEntry('maven:3.5-jdk-8-alpine', null)))
-            assertThat(params.containerMap, allOf(hasEntry('maven:3.5-jdk-8-alpine', 'maven'), hasEntry('selenium/standalone-chrome', 'selenium')))
+            assertThat(params.dockerImage, is('maven:3.5-jdk-8-alpine'))
+            assertThat(params.containerName, is('maven'))
+            assertThat(params.sidecarEnvVars, is(['testEnv': 'testVal']))
+            assertThat(params.sidecarName, is('selenium'))
+            assertThat(params.sidecarImage, is('selenium/standalone-chrome'))
             assertThat(params.containerName, is('maven'))
             assertThat(params.containerPortMappings['selenium/standalone-chrome'], hasItem(allOf(hasEntry('containerPort', 4444), hasEntry('hostPort', 4444))))
-            assertThat(params.containerWorkspaces['maven:3.5-jdk-8-alpine'], is('/home/piper'))
-            assertThat(params.containerWorkspaces['selenium/standalone-chrome'], is(''))
+            assertThat(params.dockerWorkspace, is('/home/piper'))
             body()
         })
         stepRule.step.dockerExecute(
@@ -278,10 +285,10 @@ class DockerExecuteTest extends BasePiperTest {
             dockerImage: 'maven:3.5-jdk-8-alpine',
             dockerName: 'maven',
             dockerWorkspace: '/home/piper',
-            sidecarEnvVars: ['testEnv':'testVal'],
+            sidecarEnvVars: ['testEnv': 'testVal'],
             sidecarImage: 'selenium/standalone-chrome',
             sidecarName: 'selenium',
-            sidecarVolumeBind: ['/dev/shm':'/dev/shm']
+            sidecarVolumeBind: ['/dev/shm': '/dev/shm']
         ) {
             bodyExecuted = true
         }
@@ -290,11 +297,13 @@ class DockerExecuteTest extends BasePiperTest {
     }
 
     @Test
-    void testSidecarKubernetesHealthCheck(){
+    void testSidecarKubernetesHealthCheck() {
         binding.setVariable('env', [ON_K8S: 'true'])
 
         helper.registerAllowedMethod('dockerExecuteOnKubernetes', [Map.class, Closure.class], { params, body ->
             body()
+            SidecarUtils sidecarUtils = new SidecarUtils(nullScript)
+            sidecarUtils.waitForSidecarReadyOnKubernetes(params.sidecarName, params.sidecarReadyCommand)
         })
 
         def containerCalled = false

test/groovy/GithubPublishReleaseTest.groovy

@@ -213,4 +213,19 @@ class GithubPublishReleaseTest extends BasePiperTest {
         assertThat(stepRule.step.isExcluded(item, ['won\'t fix']), is(false))
         assertJobStatusSuccess()
     }
+
+    @Test
+    void testTemplating() {
+        nullScript.commonPipelineEnvironment.setArtifactVersion('1.2.3')
+        stepRule.step.githubPublishRelease(
+            script: nullScript,
+            githubOrg: 'TestOrg',
+            githubRepo: 'TestRepo',
+            githubTokenCredentialsId: 'TestCredentials',
+            releaseBodyHeader: 'This is my release header with version: ${commonPipelineEnvironment.getArtifactVersion()} for githubOrg: ${config.githubOrg}'
+        )
+
+        assertThat('the list of closed PR is not present', data.body, containsString('This is my release header with version: 1.2.3 for githubOrg: TestOrg<br />'))
+    }
+
 }

test/groovy/HandlePipelineStepErrorsTest.groovy

@@ -86,6 +86,9 @@ class HandlePipelineStepErrorsTest extends BasePiperTest {
     @Test
     void testHandleErrorsIgnoreFailure() {
         def errorOccured = false
+        helper.registerAllowedMethod('unstable', [String.class], {s ->
+            nullScript.currentBuild.result = 'UNSTABLE'
+        })
         try {
             stepRule.step.handlePipelineStepErrors([
                 stepName: 'test',
@@ -127,6 +130,10 @@ class HandlePipelineStepErrorsTest extends BasePiperTest {
     @Test
     void testHandleErrorsIgnoreFailureNoScript() {
         def errorOccured = false
+        helper.registerAllowedMethod('unstable', [String.class], {s ->
+            //test behavior in case plugina are not yet up to date
+            throw new java.lang.NoSuchMethodError('No such DSL method \'unstable\' found')
+        })
         try {
             stepRule.step.handlePipelineStepErrors([
                 stepName: 'test',
@@ -148,6 +155,11 @@ class HandlePipelineStepErrorsTest extends BasePiperTest {
             timeout = m.time
             throw new org.jenkinsci.plugins.workflow.steps.FlowInterruptedException(hudson.model.Result.ABORTED, new jenkins.model.CauseOfInterruption.UserInterruption('Test'))
         })
+        String errorMsg
+        helper.registerAllowedMethod('unstable', [String.class], {s ->
+            nullScript.currentBuild.result = 'UNSTABLE'
+            errorMsg = s
+        })
 
         stepRule.step.handlePipelineStepErrors([
             stepName: 'test',
@@ -159,5 +171,6 @@ class HandlePipelineStepErrorsTest extends BasePiperTest {
         }
         assertThat(timeout, is(10))
         assertThat(nullScript.currentBuild.result, is('UNSTABLE'))
+        assertThat(errorMsg, is('[handlePipelineStepErrors] Error in step test - Build result set to \'UNSTABLE\''))
     }
 }

test/groovy/MailSendNotificationTest.groovy

@@ -58,10 +58,10 @@ user3@domain.com noreply+github@domain.com'''
 
     @Test
     void testCulpritsFromGitCommit() throws Exception {
-        def gitCommand = "git log -2 --pretty=format:'%ae %ce'"
+        def gitCommand = "git log -2 --first-parent --pretty=format:'%ae %ce'"
         def expected = "user2@domain.com user3@domain.com"
 
-        shellRule.setReturnValue("git log -2 --pretty=format:'%ae %ce'", 'user2@domain.com user3@domain.com')
+        shellRule.setReturnValue("git log -2 --first-parent --pretty=format:'%ae %ce'", 'user2@domain.com user3@domain.com')
 
         def result = stepRule.step.getCulprits(
             [

test/groovy/NeoDeployTest.groovy

@@ -519,8 +519,8 @@ class NeoDeployTest extends BasePiperTest {
     @Test
     void showLogsOnFailingDeployment() {
 
-        thrown.expect(Exception)
-        shellRule.failExecution(Type.REGEX, '.* deploy .*')
+        thrown.expect(AbortException)
+        shellRule.setReturnValue(Type.REGEX, '.* deploy .*', {throw new AbortException()})
 
         stepRule.step.neoDeploy(script: nullScript,
             source: warArchiveName,

test/groovy/PiperPublishWarningsTest.groovy

@@ -1,6 +1,3 @@
-#!groovy
-package steps
-
 import com.sap.piper.JenkinsUtils
 
 import static org.hamcrest.Matchers.allOf

test/groovy/SetupCommonPipelineEnvironmentTest.groovy

@@ -3,18 +3,13 @@ import org.junit.Rule
 import org.junit.Test
 import org.junit.rules.RuleChain
 import org.yaml.snakeyaml.Yaml
-
-import com.sap.piper.Utils
-
 import util.BasePiperTest
-import util.Rules
-import util.JenkinsReadYamlRule
 import util.JenkinsStepRule
+import util.Rules
 
 import static org.junit.Assert.assertEquals
 import static org.junit.Assert.assertNotNull
 
-
 class SetupCommonPipelineEnvironmentTest extends BasePiperTest {
 
     def usedConfigFile
@@ -33,7 +28,7 @@ class SetupCommonPipelineEnvironmentTest extends BasePiperTest {
 
         helper.registerAllowedMethod("readYaml", [Map], { Map parameters ->
             Yaml yamlParser = new Yaml()
-            if(parameters.text) {
+            if (parameters.text) {
                 return yamlParser.load(parameters.text)
             }
             usedConfigFile = parameters.file
@@ -55,5 +50,20 @@ class SetupCommonPipelineEnvironmentTest extends BasePiperTest {
         assertEquals('develop', nullScript.commonPipelineEnvironment.configuration.general.productiveBranch)
         assertEquals('my-maven-docker', nullScript.commonPipelineEnvironment.configuration.steps.mavenExecute.dockerImage)
     }
+
+    @Test
+    void testWorksAlsoWithYamlFileEnding() throws Exception {
+
+        helper.registerAllowedMethod("fileExists", [String], { String path ->
+            return path.endsWith('.pipeline/config.yaml')
+        })
+
+        stepRule.step.setupCommonPipelineEnvironment(script: nullScript)
+
+        assertEquals('.pipeline/config.yaml', usedConfigFile)
+        assertNotNull(nullScript.commonPipelineEnvironment.configuration)
+        assertEquals('develop', nullScript.commonPipelineEnvironment.configuration.general.productiveBranch)
+        assertEquals('my-maven-docker', nullScript.commonPipelineEnvironment.configuration.steps.mavenExecute.dockerImage)
+    }
 }
 

test/groovy/SonarExecuteScanTest.groovy

@@ -146,7 +146,7 @@ class SonarExecuteScanTest extends BasePiperTest {
             containsString('-Dsonar.pullrequest.key=42'),
             containsString('-Dsonar.pullrequest.base=master'),
             containsString('-Dsonar.pullrequest.branch=feature/anything'),
-            containsString('-Dsonar.pullrequest.provider=github'),
+            containsString('-Dsonar.pullrequest.provider=GitHub'),
             containsString('-Dsonar.pullrequest.github.repository=testOrg/testRepo')
         )))
         assertJobStatusSuccess()
@@ -234,4 +234,21 @@ class SonarExecuteScanTest extends BasePiperTest {
         assertThat(jscr.shell, hasItem(containsString('-Dsonar.organization=TestOrg-github')))
         assertJobStatusSuccess()
     }
+
+    @Test
+    void testWithCustomTlsCertificates() throws Exception {
+        jsr.step.sonarExecuteScan(
+            script: nullScript,
+            juStabUtils: utils,
+            customTlsCertificateLinks: [
+                'http://url.to/my.cert'
+            ]
+        )
+        // asserts
+        assertThat(jscr.shell, allOf(
+            hasItem(containsString('wget --directory-prefix .certificates/ --no-verbose http://url.to/my.cert')),
+            hasItem(containsString('keytool -import -noprompt -storepass changeit -keystore .sonar-scanner/jre/lib/security/cacerts -alias \'my.cert\' -file \'.certificates/my.cert\''))
+        ))
+        assertJobStatusSuccess()
+    }
 }

test/groovy/TmsUploadTest.groovy

@@ -1,5 +1,8 @@
 import com.sap.piper.JenkinsUtils
 import com.sap.piper.integration.TransportManagementService
+
+import hudson.AbortException
+
 import org.junit.After
 import org.junit.Before
 import org.junit.Rule
@@ -19,6 +22,7 @@ public class TmsUploadTest extends BasePiperTest {
     private JenkinsStepRule stepRule = new JenkinsStepRule(this)
     private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this)
     private JenkinsEnvironmentRule envRule = new JenkinsEnvironmentRule(this)
+    private JenkinsFileExistsRule fileExistsRules = new JenkinsFileExistsRule(this, ['dummy.mtar'])
 
     def tmsStub
     def jenkinsUtilsStub
@@ -56,6 +60,7 @@ public class TmsUploadTest extends BasePiperTest {
         .around(stepRule)
         .around(loggingRule)
         .around(envRule)
+        .around(fileExistsRules)
         .around(new JenkinsCredentialsRule(this)
             .withCredentials('TMS_ServiceKey', serviceKeyContent))
 
@@ -161,6 +166,27 @@ public class TmsUploadTest extends BasePiperTest {
         assertThat(loggingRule.log, containsString("[TransportManagementService] Corresponding Transport Request: 'My custom description for testing.' (Id: '2000')"))
     }
 
+    @Test
+    public void failOnMissingMtaFile() {
+
+        thrown.expect(AbortException)
+        thrown.expectMessage('Mta file \'dummy.mtar\' does not exist.')
+
+        fileExistsRules.existingFiles.remove('dummy.mtar')
+        jenkinsUtilsStub = new JenkinsUtilsMock("Test User")
+
+        stepRule.step.tmsUpload(
+            script: nullScript,
+            juStabUtils: utils,
+            jenkinsUtilsStub: jenkinsUtilsStub,
+            transportManagementService: tmsStub,
+            mtaPath: 'dummy.mtar',
+            nodeName: 'myNode',
+            credentialsId: 'TMS_ServiceKey',
+            customDescription: 'My custom description for testing.'
+        )
+    }
+
     def mockTransportManagementService() {
         return new TransportManagementService(nullScript, [:]) {
             def authentication(String uaaUrl, String oauthClientId, String oauthClientSecret) {

test/groovy/XsDeployTest.groovy

@@ -0,0 +1,480 @@
+import static org.junit.Assert.assertThat
+
+import org.hamcrest.Matchers
+
+import static org.hamcrest.Matchers.allOf
+import static org.hamcrest.Matchers.contains
+import static org.hamcrest.Matchers.containsString
+import static org.hamcrest.Matchers.hasSize
+import static org.hamcrest.Matchers.is
+
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.ExpectedException
+import org.junit.rules.RuleChain
+
+import util.BasePiperTest
+import util.CommandLineMatcher
+import util.JenkinsCredentialsRule
+import util.JenkinsDockerExecuteRule
+import util.JenkinsFileExistsRule
+import util.JenkinsLockRule
+import util.JenkinsLoggingRule
+import util.JenkinsReadYamlRule
+import util.JenkinsShellCallRule
+import util.JenkinsStepRule
+import util.Rules
+
+import com.sap.piper.JenkinsUtils
+
+import hudson.AbortException
+
+class XsDeployTest extends BasePiperTest {
+
+    private ExpectedException thrown = ExpectedException.none()
+
+    private List existingFiles =  [
+        '.xsconfig',
+        'myApp.mta'
+    ]
+
+    private JenkinsStepRule stepRule = new JenkinsStepRule(this)
+    private JenkinsShellCallRule shellRule = new JenkinsShellCallRule(this)
+    private JenkinsLockRule lockRule = new JenkinsLockRule(this)
+    private JenkinsLoggingRule logRule = new JenkinsLoggingRule(this)
+
+    @Rule
+    public RuleChain ruleChain = Rules.getCommonRules(this)
+                                        .around(new JenkinsReadYamlRule(this))
+                                        .around(stepRule)
+                                        .around(new JenkinsDockerExecuteRule(this))
+                                        .around(new JenkinsCredentialsRule(this)
+                                            .withCredentials('myCreds', 'cred_xs', 'topSecret'))
+                                        .around(new JenkinsFileExistsRule(this, existingFiles))
+                                        .around(lockRule)
+                                        .around(shellRule)
+                                        .around(logRule)
+                                        .around(thrown)
+
+    @Test
+    public void testSanityChecks() {
+
+        thrown.expect(IllegalArgumentException)
+        thrown.expectMessage(
+            allOf(
+                containsString('ERROR - NO VALUE AVAILABLE FOR:'),
+                containsString('apiUrl'),
+                containsString('org'),
+                containsString('space'),
+                containsString('mtaPath')))
+
+        stepRule.step.xsDeploy(script: nullScript)
+    }
+
+    @Test
+    public void testLoginFailed() {
+
+        thrown.expect(AbortException)
+        thrown.expectMessage('xs login failed')
+
+        shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '#!/bin/bash xs login .*', 1)
+
+        try {
+            stepRule.step.xsDeploy(
+                script: nullScript,
+                apiUrl: 'https://example.org/xs',
+                org: 'myOrg',
+                space: 'mySpace',
+                credentialsId: 'myCreds',
+                mtaPath: 'myApp.mta'
+            )
+        } catch(AbortException e ) {
+
+            assertThat(shellRule.shell,
+                allOf(
+                    // first item: the login attempt
+                    // second item: we try to provide the logs
+                    hasSize(2),
+                    new CommandLineMatcher()
+                        .hasProlog("#!/bin/bash")
+                        .hasSnippet('xs login'),
+                    new CommandLineMatcher()
+                        .hasProlog('LOG_FOLDER')
+                        .hasSnippet('cat \\$\\{LOG_FOLDER\\}/\\*')
+                )
+            )
+            throw e
+        }
+    }
+
+    @Test
+    public void testDeployFailed() {
+
+        thrown.expect(AbortException)
+        thrown.expectMessage('Failed command(s): [xs deploy]. Check earlier log for details.')
+
+        shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '#!/bin/bash.*xs deploy .*', {throw new AbortException()})
+
+        try {
+            stepRule.step.xsDeploy(
+                script: nullScript,
+                apiUrl: 'https://example.org/xs',
+                org: 'myOrg',
+                space: 'mySpace',
+                credentialsId: 'myCreds',
+                mtaPath: 'myApp.mta'
+            )
+        } catch(AbortException e ) {
+
+            assertThat(shellRule.shell,
+                allOf(
+                    hasSize(4),
+                    new CommandLineMatcher()
+                        .hasProlog("#!/bin/bash")
+                        .hasSnippet('xs login'),
+                    new CommandLineMatcher()
+                        .hasProlog("#!/bin/bash")
+                        .hasSnippet('xs deploy'),
+                    new CommandLineMatcher()
+                        .hasProlog('#!/bin/bash')
+                        .hasSnippet('xs logout'), // logout must be present in case deployment failed.
+                    new CommandLineMatcher()
+                        .hasProlog('')
+                        .hasSnippet('rm \\$\\{XSCONFIG\\}') // remove the session file after logout
+                )
+            )
+            throw e
+        }
+    }
+
+    @Test
+    public void testNothingHappensWhenModeIsNone() {
+
+        stepRule.step.xsDeploy(
+            script: nullScript,
+            mode: 'NONE'
+        )
+
+        assertThat(logRule.log, containsString('Deployment skipped intentionally.'))
+        assertThat(shellRule.shell, hasSize(0))
+    }
+
+    @Test
+    public void testDeploymentFailsWhenDeployableIsNotPresent() {
+
+        thrown.expect(AbortException)
+        thrown.expectMessage('Deployable \'myApp.mta\' does not exist.')
+
+        existingFiles.remove('myApp.mta')
+
+        try {
+            stepRule.step.xsDeploy(
+                script: nullScript,
+                apiUrl: 'https://example.org/xs',
+                org: 'myOrg',
+                space: 'mySpace',
+                credentialsId: 'myCreds',
+                mtaPath: 'myApp.mta'
+            )
+        } catch(AbortException e) {
+
+            // no shell operation happened in this case.
+            assertThat(shellRule.shell.size(), is(0))
+
+            throw e
+        }
+    }
+
+    @Test
+    public void testDeployStraighForward() {
+
+        stepRule.step.xsDeploy(
+            script: nullScript,
+            apiUrl: 'https://example.org/xs',
+            org: 'myOrg',
+            space: 'mySpace',
+            credentialsId: 'myCreds',
+            deployOpts: '-t 60',
+            mtaPath: 'myApp.mta'
+        )
+
+        assertThat(shellRule.shell,
+            allOf(
+                new CommandLineMatcher()
+                    .hasProlog("#!/bin/bash xs login")
+                    .hasSnippet('xs login')
+                    .hasOption('a', 'https://example.org/xs')
+                    .hasOption('u', 'cred_xs')
+                    .hasSingleQuotedOption('p', 'topSecret')
+                    .hasOption('o', 'myOrg')
+                    .hasOption('s', 'mySpace'),
+                new CommandLineMatcher()
+                    .hasProlog("#!/bin/bash")
+                    .hasSnippet('xs deploy')
+                    .hasOption('t', '60')
+                    .hasArgument('\'myApp.mta\''),
+                new CommandLineMatcher()
+                    .hasProlog("#!/bin/bash")
+                    .hasSnippet('xs logout')
+            )
+        )
+
+        assertThat(lockRule.getLockResources(), contains('xsDeploy:https://example.org/xs:myOrg:mySpace'))
+
+    }
+
+    @Test
+    public void testInvalidDeploymentModeProviced() {
+
+        thrown.expect(IllegalArgumentException)
+        thrown.expectMessage('No enum constant')
+
+        stepRule.step.xsDeploy(
+            script: nullScript,
+            apiUrl: 'https://example.org/xs',
+            org: 'myOrg',
+            space: 'mySpace',
+            credentialsId: 'myCreds',
+            deployOpts: '-t 60',
+            mtaPath: 'myApp.mta',
+            mode: 'DOES_NOT_EXIST'
+        )
+    }
+
+    @Test
+    public void testActionProvidedForStandardDeployment() {
+
+        thrown.expect(AbortException)
+        thrown.expectMessage(
+            'Cannot perform action \'resume\' in mode \'deploy\'. Only action \'none\' is allowed.')
+
+        stepRule.step.xsDeploy(
+            script: nullScript,
+            apiUrl: 'https://example.org/xs',
+            org: 'myOrg',
+            space: 'mySpace',
+            credentialsId: 'myCreds',
+            deployOpts: '-t 60',
+            mtaPath: 'myApp.mta',
+            mode: 'DEPLOY', // this is the default anyway
+            action: 'RESUME'
+        )
+    }
+
+    @Test
+    public void testBlueGreenDeployFailes() {
+
+        thrown.expect(AbortException)
+        thrown.expectMessage('Failed command(s): [xs bg-deploy]')
+
+        logRule.expect('Something went wrong')
+
+        shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '#!/bin/bash.*xs bg-deploy .*',
+            { throw new AbortException('Something went wrong.') })
+
+        try {
+            stepRule.step.xsDeploy(
+                script: nullScript,
+                apiUrl: 'https://example.org/xs',
+                org: 'myOrg',
+                space: 'mySpace',
+                credentialsId: 'myCreds',
+                mtaPath: 'myApp.mta',
+                mode: 'BG_DEPLOY'
+            )
+        } catch(AbortException e) {
+
+            // in case there is a deployment failure we have to logout also for bg-deployments
+            assertThat(shellRule.shell,
+                new CommandLineMatcher()
+                    .hasProlog('#!/bin/bash')
+                    .hasSnippet('xs logout')
+            )
+
+            throw e
+        }
+}
+
+    @Test
+    public void testBlueGreenDeployStraighForward() {
+
+        shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, '#!/bin/bash.*xs bg-deploy .*',
+            ((CharSequence)'''  |
+                                |
+                                |Uploading 1 files:
+                                |/myFolder/my.mtar
+                                |File upload finished
+                                |
+                                |Detected MTA schema version: "3.1.0"
+                                |Detected deploy target as "myOrg mySpace"
+                                |Detected deployed MTA with ID "my_mta" and version "0.0.1"
+                                |Deployed MTA color: blue
+                                |New MTA color: green
+                                |Detected new MTA version: "0.0.1"
+                                |Deployed MTA version: 0.0.1
+                                |Service "xxx" is not modified and will not be updated
+                                |Creating application "db-green" from MTA module "xx"...
+                                |Uploading application "xx-green"...
+                                |Staging application "xx-green"...
+                                |Application "xx-green" staged
+                                |Executing task "deploy" on application "xx-green"...
+                                |Task execution status: succeeded
+                                |Process has entered validation phase. After testing your new deployment you can resume or abort the process.
+                                |Use "xs bg-deploy -i 1234 -a resume" to resume the process.
+                                |Use "xs bg-deploy -i 1234 -a abort" to abort the process.
+                                |Hint: Use the '--no-confirm' option of the bg-deploy command to skip this phase.
+                                |''').stripMargin())
+
+        stepRule.step.xsDeploy(
+            script: nullScript,
+            apiUrl: 'https://example.org/xs',
+            org: 'myOrg',
+            space: 'mySpace',
+            credentialsId: 'myCreds',
+            deployOpts: '-t 60',
+            mtaPath: 'myApp.mta',
+            mode: 'BG_DEPLOY'
+        )
+
+        assertThat(nullScript.commonPipelineEnvironment.xsDeploymentId, is('1234'))
+
+        assertThat(shellRule.shell,
+            allOf(
+                new CommandLineMatcher()
+                    .hasProlog("#!/bin/bash xs login")
+                    .hasOption('a', 'https://example.org/xs')
+                    .hasOption('u', 'cred_xs')
+                    .hasSingleQuotedOption('p', 'topSecret')
+                    .hasOption('o', 'myOrg')
+                    .hasOption('s', 'mySpace'),
+                new CommandLineMatcher()
+                    .hasProlog("#!/bin/bash")
+                    .hasOption('t', '60')
+                    .hasArgument('\'myApp.mta\''),
+                new CommandLineMatcher()
+                    .hasProlog("#!/bin/bash")
+            )
+        )
+
+        assertThat(lockRule.getLockResources(), contains('xsDeploy:https://example.org/xs:myOrg:mySpace'))
+    }
+
+    @Test
+    public void testBlueGreenDeployResumeWithoutDeploymentId() {
+
+        // this happens in case we would like to complete a deployment without having a (successful) deployments before.
+
+        thrown.expect(IllegalArgumentException)
+        thrown.expectMessage(
+            allOf(
+                containsString('No deployment id provided'),
+                containsString('Was there a deployment before?')))
+
+        nullScript.commonPipelineEnvironment.xsDeploymentId = null // is null anyway, just for clarification
+
+        stepRule.step.xsDeploy(
+            script: nullScript,
+            apiUrl: 'https://example.org/xs',
+            org: 'myOrg',
+            space: 'mySpace',
+            credentialsId: 'myCreds',
+            mode: 'BG_DEPLOY',
+            action: 'RESUME'
+        )
+    }
+
+    @Test
+    public void testBlueGreenDeployWithoutExistingSession() {
+
+        thrown.expect(AbortException)
+        thrown.expectMessage(
+            'For the current configuration an already existing session is required.' +
+            ' But there is no already existing session')
+
+        existingFiles.remove('.xsconfig')
+
+        stepRule.step.xsDeploy(
+            script: nullScript,
+            apiUrl: 'https://example.org/xs',
+            org: 'myOrg',
+            space: 'mySpace',
+            credentialsId: 'myCreds',
+            mode: 'BG_DEPLOY',
+            action: 'RESUME'
+        )
+
+    }
+
+    @Test
+    public void testBlueGreenDeployResumeFails() {
+
+        // e.g. we try to resume a deployment which did not succeed or which was already resumed or aborted.
+
+        thrown.expect(AbortException)
+        thrown.expectMessage('Failed command(s): [xs bg-deploy -a resume].')
+
+        shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, 'xs bg-deploy -i .*', 1)
+
+        nullScript.commonPipelineEnvironment.xsDeploymentId = '1234'
+
+        try {
+            stepRule.step.xsDeploy(
+                script: nullScript,
+                apiUrl: 'https://example.org/xs',
+                org: 'myOrg',
+                space: 'mySpace',
+                credentialsId: 'myCreds',
+                mode: 'BG_DEPLOY',
+                action: 'RESUME'
+            )
+        } catch(AbortException e) {
+
+            // logout must happen also in case of a failed deployment
+            assertThat(shellRule.shell,
+                new CommandLineMatcher()
+                    .hasProlog('')
+                    .hasSnippet('xs logout'))
+            throw e
+        }
+    }
+
+    @Test
+    public void testBlueGreenDeployResume() {
+
+        shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, 'xs bg-deploy -i .*', 0)
+
+        nullScript.commonPipelineEnvironment.xsDeploymentId = '1234'
+
+        stepRule.step.xsDeploy(
+            script: nullScript,
+            apiUrl: 'https://example.org/xs',
+            org: 'myOrg',
+            space: 'mySpace',
+            credentialsId: 'myCreds',
+            mode: 'BG_DEPLOY',
+            action: 'RESUME'
+        )
+
+        // there is no login in case of a resume since we have to use the old session which triggered the deployment.
+        assertThat(shellRule.shell,
+            allOf(
+                hasSize(3),
+                new CommandLineMatcher()
+                    .hasProlog('#!/bin/bash')
+                    .hasSnippet('xs bg-deploy')
+                    .hasOption('i', '1234')
+                    .hasOption('a', 'resume'),
+                new CommandLineMatcher()
+                    .hasProlog("#!/bin/bash")
+                    .hasSnippet('xs logout'),
+                new CommandLineMatcher()
+                    .hasProlog('')
+                    .hasSnippet('rm \\$\\{XSCONFIG\\}') // delete the session file after logout
+            )
+        )
+
+        assertThat(lockRule.getLockResources(), contains('xsDeploy:https://example.org/xs:myOrg:mySpace'))
+
+    }
+
+}

test/groovy/com/sap/piper/variablesubstitution/YamlUtilsTest.groovy

@@ -0,0 +1,271 @@
+package com.sap.piper.variablesubstitution
+
+import org.junit.Before
+
+import static org.junit.Assert.*
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.ExpectedException;
+import org.junit.rules.RuleChain;
+import util.BasePiperTest
+import util.JenkinsEnvironmentRule
+import util.JenkinsErrorRule
+import util.JenkinsLoggingRule
+import util.JenkinsReadYamlRule
+import util.JenkinsWriteYamlRule
+import util.Rules
+
+class YamlUtilsTest extends BasePiperTest {
+
+    private JenkinsReadYamlRule readYamlRule = new JenkinsReadYamlRule(this)
+    private JenkinsWriteYamlRule writeYamlRule = new JenkinsWriteYamlRule(this)
+    private JenkinsErrorRule errorRule = new JenkinsErrorRule(this)
+    private JenkinsEnvironmentRule environmentRule = new JenkinsEnvironmentRule(this)
+    private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this)
+    private ExpectedException expectedExceptionRule = ExpectedException.none()
+
+    private YamlUtils yamlUtils
+
+    @Rule
+    public RuleChain rules = Rules
+        .getCommonRules(this)
+        .around(readYamlRule)
+        .around(writeYamlRule)
+        .around(errorRule)
+        .around(environmentRule)
+        .around(loggingRule)
+        .around(expectedExceptionRule)
+
+    @Before
+    public void setup() {
+        yamlUtils = new YamlUtils(nullScript)
+
+        readYamlRule.registerYaml("manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/manifest.yml")))
+                    .registerYaml("manifest-variables.yml", new FileInputStream(new File("test/resources/variableSubstitution/manifest-variables.yml")))
+                    .registerYaml("test/resources/variableSubstitution/manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/manifest.yml")))
+                    .registerYaml("test/resources/variableSubstitution/manifest-variables.yml", new FileInputStream(new File("test/resources/variableSubstitution/manifest-variables.yml")))
+                    .registerYaml("test/resources/variableSubstitution/invalid_manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/invalid_manifest.yml")))
+                    .registerYaml("test/resources/variableSubstitution/novars_manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/novars_manifest.yml")))
+                    .registerYaml("test/resources/variableSubstitution/multi_manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/multi_manifest.yml")))
+                    .registerYaml("test/resources/variableSubstitution/datatypes_manifest.yml", new FileInputStream(new File("test/resources/variableSubstitution/datatypes_manifest.yml")))
+                    .registerYaml("test/resources/variableSubstitution/datatypes_manifest-variables.yml", new FileInputStream(new File("test/resources/variableSubstitution/datatypes_manifest-variables.yml")))
+    }
+
+    @Test
+    public void substituteVariables_Fails_If_InputYamlIsNullOrEmpty() throws Exception {
+
+        expectedExceptionRule.expect(IllegalArgumentException)
+        expectedExceptionRule.expectMessage("[YamlUtils] Input Yaml data must not be null or empty.")
+
+        yamlUtils.substituteVariables(null, null)
+    }
+
+    @Test
+    public void substituteVariables_Fails_If_VariablesYamlIsNullOrEmpty() throws Exception {
+        String manifestFileName = "test/resources/variableSubstitution/manifest.yml"
+
+        expectedExceptionRule.expect(IllegalArgumentException)
+        expectedExceptionRule.expectMessage("[YamlUtils] Variables Yaml data must not be null or empty.")
+
+        Object input = nullScript.readYaml file: manifestFileName
+
+        // execute step
+        yamlUtils.substituteVariables(input, null)
+    }
+
+    @Test
+    public void substituteVariables_Throws_If_InputYamlIsInvalid() throws Exception {
+        String manifestFileName = "test/resources/variableSubstitution/invalid_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/invalid_manifest.yml"
+
+        //check that exception is thrown and that it has the correct message.
+        expectedExceptionRule.expect(org.yaml.snakeyaml.scanner.ScannerException)
+        expectedExceptionRule.expectMessage("found character '%' that cannot start any token. (Do not use % for indentation)")
+
+        Object input = nullScript.readYaml file: manifestFileName
+        Object variables = nullScript.readYaml file: variablesFileName
+
+        // execute step
+        yamlUtils.substituteVariables(input, variables)
+    }
+
+    @Test
+    public void substituteVariables_Throws_If_VariablesYamlInvalid() throws Exception {
+        String manifestFileName = "test/resources/variableSubstitution/manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/invalid_manifest.yml"
+
+        //check that exception is thrown and that it has the correct message.
+        expectedExceptionRule.expect(org.yaml.snakeyaml.scanner.ScannerException)
+        expectedExceptionRule.expectMessage("found character '%' that cannot start any token. (Do not use % for indentation)")
+
+        Object input = nullScript.readYaml file: manifestFileName
+        Object variables = nullScript.readYaml file: variablesFileName
+
+        // execute step
+        yamlUtils.substituteVariables(input, variables)
+    }
+
+    @Test
+    public void substituteVariables_ReplacesVariablesProperly_InSingleYamlFiles() throws Exception {
+        String manifestFileName = "test/resources/variableSubstitution/manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/manifest-variables.yml"
+
+        Object input = nullScript.readYaml file: manifestFileName
+        Object variables = nullScript.readYaml file: variablesFileName
+
+        // execute step
+        Map<String, Object> manifestDataAfterReplacement = yamlUtils.substituteVariables(input, variables)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        // check that resolved variables have expected values
+        assertCorrectVariableResolution(manifestDataAfterReplacement)
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    private void assertAllVariablesReplaced(String yamlStringAfterReplacement) {
+        assertFalse(yamlStringAfterReplacement.contains("(("))
+        assertFalse(yamlStringAfterReplacement.contains("))"))
+    }
+
+    private void assertCorrectVariableResolution(Map<String, Object> manifestDataAfterReplacement) {
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("name").equals("uniquePrefix-catalog-service-odatav2-0.0.1"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("routes").get(0).get("route").equals("uniquePrefix-catalog-service-odatav2-001.cfapps.eu10.hana.ondemand.com"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("services").get(0).equals("uniquePrefix-catalog-service-odatav2-xsuaa"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("services").get(1).equals("uniquePrefix-catalog-service-odatav2-hana"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("xsuaa-instance-name").equals("uniquePrefix-catalog-service-odatav2-xsuaa"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("db_service_instance_name").equals("uniquePrefix-catalog-service-odatav2-hana"))
+    }
+
+    @Test
+    public void substituteVariables_ReplacesVariablesProperly_InMultiYamlData() throws Exception {
+        String manifestFileName = "test/resources/variableSubstitution/multi_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/manifest-variables.yml"
+
+        Object input = nullScript.readYaml file: manifestFileName
+        Object variables = nullScript.readYaml file: variablesFileName
+
+        // execute step
+        List<Object> manifestDataAfterReplacement = yamlUtils.substituteVariables(input, variables)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        //check that result still is a multi-YAML file.
+        assertEquals("Dumped YAML after replacement should still be a multi-YAML file.",2, manifestDataAfterReplacement.size())
+
+        // check that resolved variables have expected values
+        manifestDataAfterReplacement.each { yaml ->
+            assertCorrectVariableResolution(yaml as Map<String, Object>)
+        }
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_ReturnsOriginalIfNoVariablesPresent() throws Exception {
+        // This test makes sure that, if no variables are found in a manifest that need
+        // to be replaced, the execution is eventually skipped and the manifest remains
+        // untouched.
+
+        String manifestFileName = "test/resources/variableSubstitution/novars_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/manifest-variables.yml"
+
+        Object input = nullScript.readYaml file: manifestFileName
+        Object variables = nullScript.readYaml file: variablesFileName
+
+        // execute step
+        ExecutionContext context = new ExecutionContext()
+        Object result = yamlUtils.substituteVariables(input, variables, context)
+
+        //Check that nothing was written
+        assertNotNull(result)
+        assertFalse(context.variablesReplaced)
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    @Test
+    public void substituteVariables_SupportsAllDataTypes() throws Exception {
+        // This test makes sure that, all datatypes supported by YAML are also
+        // properly substituted by the substituteVariables step.
+        // In particular this includes variables of type:
+        // Integer, Boolean, String, Float and inline JSON documents (which are parsed as multi-line strings)
+        // and complex types (like other YAML objects).
+        // The test also checks the differing behaviour when substituting nodes that only consist of a
+        // variable reference and nodes that contains several variable references or additional string constants.
+
+        String manifestFileName = "test/resources/variableSubstitution/datatypes_manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/datatypes_manifest-variables.yml"
+
+        Object input = nullScript.readYaml file: manifestFileName
+        Object variables = nullScript.readYaml file: variablesFileName
+
+        // execute step
+        ExecutionContext context = new ExecutionContext()
+        Map<String, Object> manifestDataAfterReplacement = yamlUtils.substituteVariables(input, variables, context)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        assertCorrectVariableResolution(manifestDataAfterReplacement)
+
+        assertDataTypeAndSubstitutionCorrectness(manifestDataAfterReplacement)
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+
+    private void assertDataTypeAndSubstitutionCorrectness(Map<String, Object> manifestDataAfterReplacement) {
+        // See datatypes_manifest.yml and datatypes_manifest-variables.yml.
+        // Note: For debugging consider turning on YAML writing to a file in JenkinsWriteYamlRule to see the
+        // actual outcome of replacing variables (for visual inspection).
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("instances").equals(1))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("instances") instanceof Integer)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("services").get(0) instanceof String)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("booleanVariable").equals(true))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("booleanVariable") instanceof Boolean)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("floatVariable") == 0.25)
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("floatVariable") instanceof Double)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("json-variable") instanceof String)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("object-variable") instanceof Map)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("string-variable").startsWith("true-0.25-1-"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("string-variable") instanceof String)
+
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("single-var-with-string-constants").equals("true-with-some-more-text"))
+        assertTrue(manifestDataAfterReplacement.get("applications").get(0).get("env").get("single-var-with-string-constants") instanceof String)
+    }
+
+    @Test
+    public void substituteVariables_DoesNotFail_If_ExecutionContextIsNull() throws Exception {
+        String manifestFileName = "test/resources/variableSubstitution/manifest.yml"
+        String variablesFileName = "test/resources/variableSubstitution/manifest-variables.yml"
+
+        Object input = nullScript.readYaml file: manifestFileName
+        Object variables = nullScript.readYaml file: variablesFileName
+
+        // execute step
+        Map<String, Object> manifestDataAfterReplacement = yamlUtils.substituteVariables(input, variables, null)
+
+        //Check that something was written
+        assertNotNull(manifestDataAfterReplacement)
+
+        // check that resolved variables have expected values
+        assertCorrectVariableResolution(manifestDataAfterReplacement)
+
+        // check that the step was marked as a success (even if it did do nothing).
+        assertJobStatusSuccess()
+    }
+}

test/groovy/templates/PiperPipelineStageInitTest.groovy

@@ -149,8 +149,7 @@ class PiperPipelineStageInitTest extends BasePiperTest {
         ]
 
         scmInfoTestList.each {scmInfoTest ->
-            jsr.step.piperPipelineStageInit.setScmInfoOnCommonPipelineEnvironment(nullScript, scmInfoTest)
-            println(scmInfoTest.GIT_URL)
+            jsr.step.piperPipelineStageInit.setGitUrlsOnCommonPipelineEnvironment(nullScript, scmInfoTest.GIT_URL)
             assertThat(nullScript.commonPipelineEnvironment.getGitSshUrl(), is(scmInfoTest.expectedSsh))
             assertThat(nullScript.commonPipelineEnvironment.getGitHttpsUrl(), is(scmInfoTest.expectedHttp))
             assertThat(nullScript.commonPipelineEnvironment.getGithubOrg(), is(scmInfoTest.expectedOrg))

test/groovy/util/CommandLineMatcher.groovy

@@ -55,7 +55,7 @@ class CommandLineMatcher extends BaseMatcher {
             }
 
             for (MapEntry opt : opts) {
-                if (!cmd.matches(/.*[\s]*--${opt.key}[\s]*${opt.value}.*/)) {
+                if (!cmd.matches(/.*[\s]*-${opt.key}[\s]*${opt.value}.*/)) {
                     hint = "A command line containing option \'${opt.key}\' with value \'${opt.value}\'"
                     matches = false
                 }

test/groovy/util/JenkinsFileExistsRule.groovy

@@ -10,11 +10,25 @@ class JenkinsFileExistsRule implements TestRule {
     final BasePipelineTest testInstance
     final List existingFiles
 
+    /**
+     * The List of files that have been queried via `fileExists`
+     */
+    final List queriedFiles = []
+
+    JenkinsFileExistsRule(BasePipelineTest testInstance) {
+        this(testInstance,[])
+    }
+
     JenkinsFileExistsRule(BasePipelineTest testInstance, List existingFiles) {
         this.testInstance = testInstance
         this.existingFiles = existingFiles
     }
 
+    JenkinsFileExistsRule registerExistingFile(String file) {
+        existingFiles.add(file)
+        return  this
+    }
+
     @Override
     Statement apply(Statement base, Description description) {
         return statement(base)
@@ -25,8 +39,15 @@ class JenkinsFileExistsRule implements TestRule {
             @Override
             void evaluate() throws Throwable {
 
-                testInstance.helper.registerAllowedMethod('fileExists', [String.class], {s -> return s in existingFiles})
-                testInstance.helper.registerAllowedMethod('fileExists', [Map.class], {m -> return m.file in existingFiles})
+                testInstance.helper.registerAllowedMethod('fileExists', [String.class], {s ->
+                    queriedFiles.add(s)
+                    return s in existingFiles
+                })
+
+                testInstance.helper.registerAllowedMethod('fileExists', [Map.class], {m ->
+                    queriedFiles.add(m.file)
+                    return m.file in existingFiles}
+                )
 
                 base.evaluate()
             }

test/groovy/util/JenkinsLoggingRule.groovy

@@ -24,8 +24,9 @@ class JenkinsLoggingRule implements TestRule {
         this.testInstance = testInstance
     }
 
-    public void expect(String substring) {
+    public JenkinsLoggingRule expect(String substring) {
         expected.add(substring)
+        return this
     }
 
     @Override

test/groovy/util/JenkinsReadYamlRule.groovy

@@ -42,11 +42,37 @@ class JenkinsReadYamlRule implements TestRule {
                     } else {
                         throw new IllegalArgumentException("Key 'text' and 'file' are both missing in map ${m}.")
                     }
-                    return new Yaml().load(yml)
+
+
+                    return readYaml(yml)
                 })
 
                 base.evaluate()
             }
         }
     }
+
+    /**
+     * Mimicking code of the original library (link below).
+     * <p>
+     * Yaml files may contain several YAML sections, separated by ---.
+     * This loads them all and returns a {@code List} of entries in case multiple sections were found or just
+     * a single {@code Object}, if only one section was read.
+     * @see https://github.com/jenkinsci/pipeline-utility-steps-plugin/blob/master/src/main/java/org/jenkinsci/plugins/pipeline/utility/steps/conf/ReadYamlStep.java
+     */
+    private def readYaml(def yml) {
+        Iterable<Object> yaml = new Yaml().loadAll(yml)
+
+        List<Object> result = new LinkedList<Object>()
+        for (Object data : yaml) {
+            result.add(data)
+        }
+
+        // If only one YAML document, return it directly
+        if (result.size() == 1) {
+            return result.get(0);
+        }
+
+        return result;
+    }
 }

test/groovy/util/JenkinsShellCallRule.groovy

@@ -42,7 +42,6 @@ class JenkinsShellCallRule implements TestRule {
     List shell = []
 
     Map<Command, String> returnValues = [:]
-    List<Command> failingCommands = []
 
     JenkinsShellCallRule(BasePipelineTest testInstance) {
         this.testInstance = testInstance
@@ -56,8 +55,28 @@ class JenkinsShellCallRule implements TestRule {
         returnValues[new Command(type, script)] = value
     }
 
-    def failExecution(type, script) {
-        failingCommands.add(new Command(type, script))
+    def handleShellCall(Map parameters) {
+
+        def unifiedScript = unify(parameters.script)
+
+        shell.add(unifiedScript)
+
+        def result = null
+
+        for(def e : returnValues.entrySet()) {
+            if(e.key.type == Type.REGEX && unifiedScript =~ e.key.script) {
+                result =  e.value
+                break
+        } else if(e.key.type == Type.PLAIN && unifiedScript.equals(e.key.script)) {
+                result = e.value
+                break
+            }
+        }
+        if(result instanceof Closure) result = result()
+        if (!result && parameters.returnStatus) result = 0
+
+        if(! parameters.returnStdout && ! parameters.returnStatus) return
+        return result
     }
 
     @Override
@@ -71,53 +90,15 @@ class JenkinsShellCallRule implements TestRule {
             void evaluate() throws Throwable {
 
                 testInstance.helper.registerAllowedMethod("sh", [String.class], {
-                    command ->
-                        def unifiedScript = unify(command)
-
-                        shell.add(unifiedScript)
-
-                        for (Command failingCommand: failingCommands){
-                            if(failingCommand.type == Type.REGEX && unifiedScript =~ failingCommand.script) {
-                                throw new Exception("Script execution failed!")
-                                break
-                            } else if(failingCommand.type == Type.PLAIN && unifiedScript.equals(failingCommand.script)) {
-                                throw new Exception("Script execution failed!")
-                                break
-                            }
-                        }
+                    command -> handleShellCall([
+                        script: command,
+                        returnStdout: false,
+                        returnStatus: false
+                    ])
                 })
 
                 testInstance.helper.registerAllowedMethod("sh", [Map.class], {
-                    m ->
-                        shell.add(m.script.replaceAll(/\s+/," ").trim())
-
-                        def unifiedScript = unify(m.script)
-                        for (Command failingCommand: failingCommands){
-                            if(failingCommand.type == Type.REGEX && unifiedScript =~ failingCommand.script) {
-                                throw new Exception("Script execution failed!")
-                                break
-                            } else if(failingCommand.type == Type.PLAIN && unifiedScript.equals(failingCommand.script)) {
-                                throw new Exception("Script execution failed!")
-                                break
-                            }
-                        }
-
-                        if (m.returnStdout || m.returnStatus) {
-                            def result = null
-
-                            for(def e : returnValues.entrySet()) {
-                                if(e.key.type == Type.REGEX && unifiedScript =~ e.key.script) {
-                                    result =  e.value
-                                    break
-                                } else if(e.key.type == Type.PLAIN && unifiedScript.equals(e.key.script)) {
-                                    result = e.value
-                                    break
-                                }
-                            }
-                            if(result instanceof Closure) result = result()
-                            if (!result && m.returnStatus) result = 0
-                            return result
-                        }
+                    m -> handleShellCall(m)
                 })
 
                 base.evaluate()

test/groovy/util/JenkinsWriteYamlRule.groovy

@@ -0,0 +1,59 @@
+package util
+
+import com.lesfurets.jenkins.unit.BasePipelineTest
+import org.yaml.snakeyaml.Yaml
+
+import static org.junit.Assert.*
+import org.junit.rules.TestRule
+import org.junit.runner.Description
+import org.junit.runners.model.Statement
+
+class JenkinsWriteYamlRule implements TestRule {
+
+    final BasePipelineTest testInstance
+    static final String DATA = "DATA" // key in files map to retrieve Yaml object graph data..
+    static final String CHARSET = "CHARSET" // key in files map to retrieve the charset of the serialized Yaml.
+    static final String SERIALIZED_YAML = "SERIALIZED_YAML" // key in files map to retrieve serialized Yaml.
+
+    Map<String, Map<String, Object>> files = new HashMap<>()
+
+    JenkinsWriteYamlRule(BasePipelineTest testInstance) {
+        this.testInstance = testInstance
+    }
+
+    @Override
+    Statement apply(Statement base, Description description) {
+        return statement(base)
+    }
+
+    private Statement statement(final Statement base) {
+        return new Statement() {
+            @Override
+            void evaluate() throws Throwable {
+
+                testInstance.helper.registerAllowedMethod( 'writeYaml', [Map], { parameterMap ->
+                    assertNotNull(parameterMap.file)
+                    assertNotNull(parameterMap.data)
+                    // charset is optional.
+
+                    Yaml yaml = new Yaml()
+                    StringWriter writer = new StringWriter()
+                    yaml.dump(parameterMap.data, writer)
+
+                    // Enable this to actually produce a file.
+                    // yaml.dump(parameterMap.data, new FileWriter(parameterMap.file))
+                    // yaml.dump(parameterMap.data, new FileWriter("test/resources/variableSubstitution/manifest_out.yml"))
+
+                    Map<String, Object> details = new HashMap<>()
+                    details.put(DATA, parameterMap.data)
+                    details.put(CHARSET, parameterMap.charset ?: "UTF-8")
+                    details.put(SERIALIZED_YAML, writer.toString())
+
+                    files[parameterMap.file] = details
+                })
+
+                base.evaluate()
+            }
+        }
+    }
+}

test/resources/variableSubstitution/datatypes_manifest-variables.yml

@@ -0,0 +1,19 @@
+---
+unique-prefix: uniquePrefix # A unique prefix. E.g. your D/I/C-User
+xsuaa-instance-name: uniquePrefix-catalog-service-odatav2-xsuaa
+hana-instance-name: uniquePrefix-catalog-service-odatav2-hana
+integer-variable: 1
+boolean-variable: Yes
+float-variable: 0.25
+json-variable: >
+  [
+    {"name":"token-destination",
+     "url":"https://www.google.com",
+     "forwardAuthToken": true}
+  ]
+object-variable:
+  hello: "world"
+  this:  "is an object with"
+  one: 1
+  float: 25.0
+  bool: Yes

test/resources/variableSubstitution/datatypes_manifest.yml

@@ -0,0 +1,27 @@
+---
+applications:
+- name: ((unique-prefix))-catalog-service-odatav2-0.0.1
+  memory: 1024M
+  disk_quota: 512M
+  instances: ((integer-variable))
+  buildpacks:
+    - java_buildpack
+  path: ./srv/target/srv-backend-0.0.1-SNAPSHOT.jar
+
+  routes:
+  - route: ((unique-prefix))-catalog-service-odatav2-001.cfapps.eu10.hana.ondemand.com
+
+  services:
+  - ((xsuaa-instance-name)) # requires an instance of xsuaa instantiated with xs-security.json of this project. See services-manifest.yml.
+  - ((hana-instance-name))  # requires an instance of hana service with plan hdi-shared. See services-manifest.yml.
+
+  env:
+    spring.profiles.active: cloud # activate the spring profile named 'cloud'.
+    xsuaa-instance-name: ((xsuaa-instance-name))
+    db_service_instance_name: ((hana-instance-name))
+    booleanVariable: ((boolean-variable))
+    floatVariable: ((float-variable))
+    json-variable: ((json-variable))
+    object-variable: ((object-variable))
+    string-variable: ((boolean-variable))-((float-variable))-((integer-variable))-((json-variable))
+    single-var-with-string-constants: ((boolean-variable))-with-some-more-text

test/resources/variableSubstitution/invalid_manifest.yml

@@ -0,0 +1,2 @@
+---
+test: %invalid

test/resources/variableSubstitution/manifest-variables-conflicting.yml

@@ -0,0 +1,4 @@
+---
+unique-prefix: uniquePrefix-conflicting-from-file # A unique prefix. E.g. your D/I/C-User
+xsuaa-instance-name: uniquePrefix-catalog-service-odatav2-xsuaa-conflicting-from-file
+

test/resources/variableSubstitution/manifest-variables.yml

@@ -0,0 +1,4 @@
+---
+unique-prefix: uniquePrefix # A unique prefix. E.g. your D/I/C-User
+xsuaa-instance-name: uniquePrefix-catalog-service-odatav2-xsuaa
+hana-instance-name: uniquePrefix-catalog-service-odatav2-hana

test/resources/variableSubstitution/manifest.yml

@@ -0,0 +1,21 @@
+---
+applications:
+- name: ((unique-prefix))-catalog-service-odatav2-0.0.1
+  memory: 1024M
+  disk_quota: 512M
+  instances: 1
+  buildpacks:
+    - java_buildpack
+  path: ./srv/target/srv-backend-0.0.1-SNAPSHOT.jar
+  
+  routes: 
+  - route: ((unique-prefix))-catalog-service-odatav2-001.cfapps.eu10.hana.ondemand.com
+
+  services:
+  - ((xsuaa-instance-name)) # requires an instance of xsuaa instantiated with xs-security.json of this project. See services-manifest.yml.
+  - ((hana-instance-name))  # requires an instance of hana service with plan hdi-shared. See services-manifest.yml.
+  
+  env:
+    spring.profiles.active: cloud # activate the spring profile named 'cloud'.
+    xsuaa-instance-name: ((xsuaa-instance-name))
+    db_service_instance_name: ((hana-instance-name))

test/resources/variableSubstitution/multi_manifest.yml

@@ -0,0 +1,43 @@
+---
+applications:
+- name: ((unique-prefix))-catalog-service-odatav2-0.0.1
+  memory: 1024M
+  disk_quota: 512M
+  instances: 1
+  buildpacks:
+    - java_buildpack
+  path: ./srv/target/srv-backend-0.0.1-SNAPSHOT.jar
+
+  routes:
+  - route: ((unique-prefix))-catalog-service-odatav2-001.cfapps.eu10.hana.ondemand.com
+
+  services:
+  - ((xsuaa-instance-name)) # requires an instance of xsuaa instantiated with xs-security.json of this project. See services-manifest.yml.
+  - ((hana-instance-name))  # requires an instance of hana service with plan hdi-shared. See services-manifest.yml.
+
+  env:
+    spring.profiles.active: cloud # activate the spring profile named 'cloud'.
+    xsuaa-instance-name: ((xsuaa-instance-name))
+    db_service_instance_name: ((hana-instance-name))
+
+---
+applications:
+  - name: ((unique-prefix))-catalog-service-odatav2-0.0.1
+    memory: 1024M
+    disk_quota: 512M
+    instances: 1
+    buildpacks:
+      - java_buildpack
+    path: ./srv/target/srv-backend-0.0.1-SNAPSHOT.jar
+
+    routes:
+      - route: ((unique-prefix))-catalog-service-odatav2-001.cfapps.eu10.hana.ondemand.com
+
+    services:
+      - ((xsuaa-instance-name)) # requires an instance of xsuaa instantiated with xs-security.json of this project. See services-manifest.yml.
+      - ((hana-instance-name))  # requires an instance of hana service with plan hdi-shared. See services-manifest.yml.
+
+    env:
+      spring.profiles.active: cloud # activate the spring profile named 'cloud'.
+      xsuaa-instance-name: ((xsuaa-instance-name))
+      db_service_instance_name: ((hana-instance-name))

test/resources/variableSubstitution/novars_manifest.yml

@@ -0,0 +1,21 @@
+---
+applications:
+- name: test-catalog-service-odatav2-0.0.1
+  memory: 1024M
+  disk_quota: 512M
+  instances: 1
+  buildpacks:
+    - java_buildpack
+  path: ./srv/target/srv-backend-0.0.1-SNAPSHOT.jar
+
+  routes:
+  - route: test-catalog-service-odatav2-001.cfapps.eu10.hana.ondemand.com
+
+  services:
+  - xsuaa-instance-name # requires an instance of xsuaa instantiated with xs-security.json of this project. See services-manifest.yml.
+  - hana-instance-name  # requires an instance of hana service with plan hdi-shared. See services-manifest.yml.
+
+  env:
+    spring.profiles.active: cloud # activate the spring profile named 'cloud'.
+    xsuaa-instance-name: xsuaa-instance-name
+    db_service_instance_name: hana-instance-name

vars/artifactSetVersion.groovy

@@ -7,7 +7,7 @@ import com.sap.piper.Utils
 import com.sap.piper.versioning.ArtifactVersioning
 
 import groovy.transform.Field
-import groovy.text.SimpleTemplateEngine
+import groovy.text.GStringTemplateEngine
 
 @Field String STEP_NAME = getClass().getName()
 @Field Map CONFIG_KEY_COMPATIBILITY = [gitSshKeyCredentialsId: 'gitCredentialsId']
@@ -144,7 +144,7 @@ void call(Map parameters = [:], Closure body = null) {
             newVersion = currentVersion
         } else {
             def binding = [version: currentVersion, timestamp: config.timestamp, commitId: config.gitCommitId]
-            newVersion = new SimpleTemplateEngine().createTemplate(config.versioningTemplate).make(binding).toString()
+            newVersion = new GStringTemplateEngine().createTemplate(config.versioningTemplate).make(binding).toString()
         }
 
         artifactVersioning.setVersion(newVersion)

vars/batsExecuteTests.groovy

@@ -5,7 +5,7 @@ import com.sap.piper.ConfigurationHelper
 import com.sap.piper.GitUtils
 import com.sap.piper.Utils
 import com.sap.piper.analytics.InfluxData
-import groovy.text.SimpleTemplateEngine
+import groovy.text.GStringTemplateEngine
 import groovy.transform.Field
 
 @Field String STEP_NAME = getClass().getName()
@@ -86,7 +86,7 @@ void call(Map parameters = [:]) {
         //resolve commonPipelineEnvironment references in envVars
         config.envVarList = []
         config.envVars.each {e ->
-            def envValue = SimpleTemplateEngine.newInstance().createTemplate(e.getValue()).make(commonPipelineEnvironment: script.commonPipelineEnvironment).toString()
+            def envValue = GStringTemplateEngine.newInstance().createTemplate(e.getValue()).make(commonPipelineEnvironment: script.commonPipelineEnvironment).toString()
             config.envVarList.add("${e.getKey()}=${envValue}")
         }
 

vars/buildExecute.groovy

@@ -3,7 +3,7 @@ import com.sap.piper.GenerateDocumentation
 import com.sap.piper.Utils
 import com.sap.piper.ConfigurationHelper
 
-import groovy.text.SimpleTemplateEngine
+import groovy.text.GStringTemplateEngine
 import groovy.transform.Field
 
 import static com.sap.piper.Prerequisites.checkScript

vars/cfManifestSubstituteVariables.groovy

@@ -0,0 +1,264 @@
+import com.sap.piper.ConfigurationHelper
+import com.sap.piper.GenerateDocumentation
+import com.sap.piper.variablesubstitution.ExecutionContext
+import com.sap.piper.variablesubstitution.DebugHelper
+import com.sap.piper.variablesubstitution.YamlUtils
+import groovy.transform.Field
+
+import static com.sap.piper.Prerequisites.checkScript
+
+@Field String STEP_NAME = getClass().getName()
+@Field Set GENERAL_CONFIG_KEYS = []
+@Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS + [
+    /**
+     * The `String` path of the Yaml file to replace variables in.
+     * Defaults to "manifest.yml" if not specified otherwise.
+     */
+    'manifestFile',
+    /**
+     * The `String` path of the Yaml file to produce as output.
+     * If not specified this will default to `manifestFile` and overwrite it.
+     */
+    'outputManifestFile',
+    /**
+     * The `List` of `String` paths of the Yaml files containing the variable values to use as a replacement in the manifest file.
+     * Defaults to `["manifest-variables.yml"]` if not specified otherwise. The order of the files given in the list is relevant
+     * in case there are conflicting variable names and values within variable files. In such a case, the values of the last file win.
+     */
+    'manifestVariablesFiles',
+    /**
+     * A `List` of `Map` entries for key-value pairs used for variable substitution within the file given by `manifestFile`.
+     * Defaults to an empty list, if not specified otherwise. This can be used to set variables like it is provided
+     * by `cf push --var key=value`.
+     *
+     * The order of the maps of variables given in the list is relevant in case there are conflicting variable names and values
+     * between maps contained within the list. In case of conflicts, the last specified map in the list will win.
+     *
+     * Though each map entry in the list can contain more than one key-value pair for variable substitution, it is recommended
+     * to stick to one entry per map, and rather declare more maps within the list. The reason is that
+     * if a map in the list contains more than one key-value entry, and the entries are conflicting, the
+     * conflict resolution behavior is undefined (since map entries have no sequence).
+     *
+     * Note: variables defined via `manifestVariables` always win over conflicting variables defined via any file given
+     * by `manifestVariablesFiles` - no matter what is declared before. This reproduces the same behavior as can be
+     * observed when using `cf push --var` in combination with `cf push --vars-file`.
+     */
+    'manifestVariables'
+]
+
+@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS
+
+/*
+ * Step to substitute variables in a given YAML file with those specified in one or more variables files given by the
+ * `manifestVariablesFiles` parameter. This follows the behavior of `cf push --vars-file`, and can be
+ * used as a pre-deployment step if commands other than `cf push` are used for deployment (e.g. `cf blue-green-deploy`).
+ *
+ * The format to reference a variable in the manifest YAML file is to use double parentheses `((` and `))`, e.g. `((variableName))`.
+ *
+ * You can declare variable assignments as key value-pairs inside a YAML variables file following the
+ * [Cloud Foundry standards](https://docs.cloudfoundry.org/devguide/deploy-apps/manifest-attributes.html#variable-substitution) format.
+ *
+ * Optionally, you can also specify a direct list of key-value mappings for variables using the `manifestVariables` parameter.
+ * Variables given in the `manifestVariables` list will take precedence over those found in variables files. This follows
+ * the behavior of `cf push --var`, and works in combination with `manifestVariablesFiles`.
+ *
+ * The step is activated by the presence of the file specified by the `manifestFile` parameter and all variables files
+ * specified by the `manifestVariablesFiles` parameter, or if variables are passed in directly via `manifestVariables`.
+ *
+ * In case no `manifestVariablesFiles` were explicitly specified, a default named `manifest-variables.yml` will be looked
+ * for and if present will activate this step also. This is to support convention over configuration.
+ */
+@GenerateDocumentation
+void call(Map arguments = [:]) {
+    handlePipelineStepErrors (stepName: STEP_NAME, stepParameters: arguments) {
+        def script = checkScript(this, arguments)  ?: this
+
+        // load default & individual configuration
+        Map config = ConfigurationHelper.newInstance(this)
+                                        .loadStepDefaults()
+                                        .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS)
+                                        .mixinStageConfig(script.commonPipelineEnvironment, arguments.stageName ?: env.STAGE_NAME, STEP_CONFIG_KEYS)
+                                        .mixin(arguments, PARAMETER_KEYS)
+                                        .use()
+
+        String defaultManifestFileName = "manifest.yml"
+        String defaultManifestVariablesFileName = "manifest-variables.yml"
+
+        Boolean manifestVariablesFilesExplicitlySpecified = config.manifestVariablesFiles != null
+
+        String manifestFilePath = config.manifestFile ?: defaultManifestFileName
+        List<String> manifestVariablesFiles = (config.manifestVariablesFiles != null) ? config.manifestVariablesFiles : [ defaultManifestVariablesFileName ]
+        List<Map<String, Object>> manifestVariablesList = config.manifestVariables ?: []
+        String outputFilePath = config.outputManifestFile ?: manifestFilePath
+
+        DebugHelper debugHelper = new DebugHelper(script, config)
+        YamlUtils yamlUtils = new YamlUtils(script, debugHelper)
+
+        Boolean manifestExists = fileExists manifestFilePath
+        Boolean manifestVariablesFilesExist = allManifestVariableFilesExist(manifestVariablesFiles)
+        Boolean manifestVariablesListSpecified = !manifestVariablesList.isEmpty()
+
+        if (!manifestExists) {
+            echo "[CFManifestSubstituteVariables] Could not find YAML file at ${manifestFilePath}. Skipping variable substitution."
+            return
+        }
+
+        if (!manifestVariablesFilesExist && manifestVariablesFilesExplicitlySpecified) {
+            // If the user explicitly specified a list of variables files, make sure they all exist.
+            // Otherwise throw an error so the user knows that he / she made a mistake.
+            error "[CFManifestSubstituteVariables] Could not find all given manifest variable substitution files. Make sure all files given as manifestVariablesFiles exist."
+        }
+
+        def result
+        ExecutionContext context = new ExecutionContext()
+
+        if (!manifestVariablesFilesExist && !manifestVariablesFilesExplicitlySpecified) {
+            // If no variables files exist (not even the default one) we check if at least we have a list of variables.
+
+            if (!manifestVariablesListSpecified) {
+                // If we have no variable values to replace references with, we skip substitution.
+                echo "[CFManifestSubstituteVariables] Could not find any default manifest variable substitution file at ${defaultManifestVariablesFileName}, and no manifest variables list was specified. Skipping variable substitution."
+                return
+            }
+
+            // If we have a list of variables specified, we can start replacing them...
+            result = substitute(manifestFilePath, [], manifestVariablesList, yamlUtils, context, debugHelper)
+        }
+        else {
+            // If we have at least one existing variable substitution file, we can start replacing variables...
+            result = substitute(manifestFilePath, manifestVariablesFiles, manifestVariablesList, yamlUtils, context, debugHelper)
+        }
+
+        if (!context.variablesReplaced) {
+            // If no variables have been replaced at all, we skip writing a file.
+            echo "[CFManifestSubstituteVariables] No variables were found or could be replaced in ${manifestFilePath}. Skipping variable substitution."
+            return
+        }
+
+        // writeYaml won't overwrite the file. You need to delete it first.
+        deleteFile(outputFilePath)
+
+        writeYaml file: outputFilePath, data: result
+
+        echo "[CFManifestSubstituteVariables] Replaced variables in ${manifestFilePath}."
+        echo "[CFManifestSubstituteVariables] Wrote output file (with variables replaced) at ${outputFilePath}."
+    }
+}
+
+/*
+ * Substitutes variables specified in files and as lists in a given manifest file.
+ * @param manifestFilePath - the path to the manifest file to replace variables in.
+ * @param manifestVariablesFiles - the paths to variables substitution files.
+ * @param manifestVariablesList - the list of variables data to replace variables with.
+ * @param yamlUtils - the `YamlUtils` used for variable substitution.
+ * @param context - an `ExecutionContext` to examine if any variables have been replaced and should be written.
+ * @param debugHelper - a debug output helper.
+ * @return an Object graph of Yaml data with variables substituted (if any were found and could be replaced).
+ */
+private Object substitute(String manifestFilePath, List<String> manifestVariablesFiles, List<Map<String, Object>> manifestVariablesList, YamlUtils yamlUtils, ExecutionContext context, DebugHelper debugHelper) {
+    Boolean noVariablesReplaced = true
+
+    def manifestData = loadManifestData(manifestFilePath, debugHelper)
+
+    // replace variables from list first.
+    List<Map<String>> reversedManifestVariablesList = manifestVariablesList.reverse() // to make sure last one wins.
+
+    def result = manifestData
+    for (Map<String, Object> manifestVariableData : reversedManifestVariablesList) {
+        def executionContext = new ExecutionContext()
+        result = yamlUtils.substituteVariables(result, manifestVariableData, executionContext)
+        noVariablesReplaced = noVariablesReplaced && !executionContext.variablesReplaced // remember if variables were replaced.
+    }
+
+    // replace remaining variables from files
+    List<String> reversedManifestVariablesFilesList = manifestVariablesFiles.reverse() // to make sure last one wins.
+    for (String manifestVariablesFilePath : reversedManifestVariablesFilesList) {
+        def manifestVariablesFileData = loadManifestVariableFileData(manifestVariablesFilePath, debugHelper)
+        def executionContext = new ExecutionContext()
+        result = yamlUtils.substituteVariables(result, manifestVariablesFileData, executionContext)
+        noVariablesReplaced = noVariablesReplaced && !executionContext.variablesReplaced // remember if variables were replaced.
+    }
+
+    context.variablesReplaced = !noVariablesReplaced
+    return result
+}
+
+/*
+ * Loads the contents of a manifest.yml file by parsing Yaml and returning the
+ * object graph. May return a `List<Object>`  (in case more YAML segments are in the file)
+ * or a `Map<String, Object>` in case there is just one segment.
+ * @param manifestFilePath - the file path of the manifest to parse.
+ * @param debugHelper - a debug output helper.
+ * @return the parsed object graph.
+ */
+private Object loadManifestData(String manifestFilePath, DebugHelper debugHelper) {
+    try {
+        // may return a List<Object>  (in case more YAML segments are in the file)
+        // or a Map<String, Object> in case there is just one segment.
+        def result = readYaml file: manifestFilePath
+        echo "[CFManifestSubstituteVariables] Loaded manifest at ${manifestFilePath}!"
+        return result
+    }
+    catch(Exception ex) {
+        debugHelper.debug("Exception: ${ex}")
+        echo "[CFManifestSubstituteVariables] Could not load manifest file at ${manifestFilePath}. Exception was: ${ex}"
+        throw ex
+    }
+}
+
+/*
+ * Loads the contents of a manifest variables file by parsing Yaml and returning the
+ * object graph. May return a `List<Object>`  (in case more YAML segments are in the file)
+ * or a `Map<String, Object>` in case there is just one segment.
+ * @param variablesFilePath - the path to the variables file to parse.
+ * @param debugHelper - a debug output helper.
+ * @return the parsed object graph.
+ */
+private Object loadManifestVariableFileData(String variablesFilePath, DebugHelper debugHelper) {
+    try {
+        // may return a List<Object>  (in case more YAML segments are in the file)
+        // or a Map<String, Object> in case there is just one segment.
+        def result = readYaml file: variablesFilePath
+        echo "[CFManifestSubstituteVariables] Loaded variables file at ${variablesFilePath}!"
+        return result
+    }
+    catch(Exception ex) {
+        debugHelper.debug("Exception: ${ex}")
+        echo "[CFManifestSubstituteVariables] Could not load manifest variables file at ${variablesFilePath}. Exception was: ${ex}"
+        throw ex
+    }
+}
+
+/*
+ * Checks if all file paths given in the list exist as files.
+ * @param manifestVariablesFiles - the list of file paths pointing to manifest variables files.
+ * @return `true`, if all given files exist, `false` otherwise.
+ */
+private boolean allManifestVariableFilesExist(List<String> manifestVariablesFiles) {
+    for (String filePath : manifestVariablesFiles) {
+        Boolean fileExists = fileExists filePath
+        if (!fileExists) {
+            echo "[CFManifestSubstituteVariables] Did not find manifest variable substitution file at ${filePath}."
+            return false
+        }
+    }
+    return true
+}
+
+/*
+ * Removes the given file, if it exists.
+ * @param filePath - the path to the file to remove.
+ */
+private void deleteFile(String filePath) {
+
+    Boolean fileExists = fileExists file: filePath
+    if(fileExists) {
+        Boolean failure = sh script: "rm '${filePath}'", returnStatus: true
+        if(!failure) {
+            echo "[CFManifestSubstituteVariables] Successfully deleted file '${filePath}'."
+        }
+        else {
+            error "[CFManifestSubstituteVariables] Could not delete file '${filePath}'. Check file permissions."
+        }
+    }
+}

vars/checksPublishResults.groovy

@@ -181,7 +181,6 @@ def createCommonOptionsMap(publisherName, settings){
     return result
 }
 
-@NonCPS
 def prepare(parameters){
     // ensure tool maps are initialized correctly
     for(String tool : TOOLS){
@@ -190,7 +189,6 @@ def prepare(parameters){
     return parameters
 }
 
-@NonCPS
 def toMap(parameter){
     if(MapUtils.isMap(parameter))
         parameter.put('active', parameter.active == null?true:parameter.active)

vars/cloudFoundryCreateService.groovy

@@ -0,0 +1,179 @@
+import com.sap.piper.GenerateDocumentation
+import com.sap.piper.BashUtils
+import com.sap.piper.JenkinsUtils
+import com.sap.piper.Utils
+import com.sap.piper.ConfigurationHelper
+
+import groovy.transform.Field
+
+import static com.sap.piper.Prerequisites.checkScript
+
+@Field String STEP_NAME = 'cloudFoundryCreateService'
+
+@Field Set STEP_CONFIG_KEYS = [
+    'cloudFoundry',
+        /**
+         * Cloud Foundry API endpoint.
+         * @parentConfigKey cloudFoundry
+         */
+        'apiEndpoint',
+        /**
+         * Credentials to be used for deployment.
+         * @parentConfigKey cloudFoundry
+         */
+        'credentialsId',
+        /**
+         * Defines the manifest Yaml file that contains the information about the to be created services that will be passed to a Create-Service-Push cf cli plugin.
+         * @parentConfigKey cloudFoundry
+         */
+        'serviceManifest',
+        /**
+         * Defines the manifest variables Yaml files to be used to replace variable references in manifest. This parameter
+         * is optional and will default to `["manifest-variables.yml"]`. This can be used to set variable files like it
+         * is provided by `cf push --vars-file <file>`.
+         *
+         * If the manifest is present and so are all variable files, a variable substitution will be triggered that uses
+         * the `cfManifestSubstituteVariables` step before deployment. The format of variable references follows the
+         * [Cloud Foundry standard](https://docs.cloudfoundry.org/devguide/deploy-apps/manifest-attributes.html#variable-substitution).
+         * @parentConfigKey cloudFoundry
+         */
+        'manifestVariablesFiles',
+        /**
+         * Defines a `List` of variables as key-value `Map` objects used for variable substitution within the file given by `manifest`.
+         * Defaults to an empty list, if not specified otherwise. This can be used to set variables like it is provided
+         * by `cf push --var key=value`.
+         *
+         * The order of the maps of variables given in the list is relevant in case there are conflicting variable names and values
+         * between maps contained within the list. In case of conflicts, the last specified map in the list will win.
+         *
+         * Though each map entry in the list can contain more than one key-value pair for variable substitution, it is recommended
+         * to stick to one entry per map, and rather declare more maps within the list. The reason is that
+         * if a map in the list contains more than one key-value entry, and the entries are conflicting, the
+         * conflict resolution behavior is undefined (since map entries have no sequence).
+         *
+         * Note: variables defined via `manifestVariables` always win over conflicting variables defined via any file given
+         * by `manifestVariablesFiles` - no matter what is declared before. This is the same behavior as can be
+         * observed when using `cf push --var` in combination with `cf push --vars-file`.
+         */
+        'manifestVariables',
+        /**
+         * Cloud Foundry target organization.
+         * @parentConfigKey cloudFoundry
+         */
+        'org',
+        /**
+         * Cloud Foundry target space.
+         * @parentConfigKey cloudFoundry
+         */
+        'space',
+    /** @see dockerExecute */
+    'dockerImage',
+    /** @see dockerExecute */
+    'dockerWorkspace',
+    /** @see dockerExecute */
+    'stashContent'
+]
+
+@Field Map CONFIG_KEY_COMPATIBILITY = [cloudFoundry: [apiEndpoint: 'cfApiEndpoint', appName:'cfAppName', credentialsId: 'cfCredentialsId', serviceManifest: 'cfServiceManifest', manifestVariablesFiles: 'cfManifestVariablesFiles', manifestVariables: 'cfManifestVariables',  org: 'cfOrg', space: 'cfSpace']]
+@Field Set GENERAL_CONFIG_KEYS = STEP_CONFIG_KEYS
+@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS
+
+/**
+ * Uses the Create-Service-Push plugin to create services in a Cloud Foundry space.
+ *
+ * For details how to specify the services see the [github page of the plugin](https://github.com/dawu415/CF-CLI-Create-Service-Push-Plugin).
+ *
+ * The `--no-push` options is always used with the plugin. To deploy the application make use of the cloudFoundryDeploy step!
+ */
+@GenerateDocumentation
+void call(Map parameters = [:]) {
+    handlePipelineStepErrors (stepName: STEP_NAME, stepParameters: parameters) {
+        def script = checkScript(this, parameters) ?: this
+        def utils = parameters.juStabUtils ?: new Utils()
+        def jenkinsUtils = parameters.jenkinsUtilsStub ?: new JenkinsUtils()
+        // load default & individual configuration
+        Map config = ConfigurationHelper.newInstance(this)
+            .loadStepDefaults()
+            .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY)
+            .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY)
+            .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName?:env.STAGE_NAME, STEP_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY)
+            .mixin(parameters, PARAMETER_KEYS, CONFIG_KEY_COMPATIBILITY)
+            .withMandatoryProperty('cloudFoundry/org')
+            .withMandatoryProperty('cloudFoundry/space')
+            .withMandatoryProperty('cloudFoundry/credentialsId')
+            .withMandatoryProperty('cloudFoundry/serviceManifest')
+            .use()
+
+
+        utils.pushToSWA([step: STEP_NAME],config)
+
+        utils.unstashAll(config.stashContent)
+
+        if (fileExists(config.cloudFoundry.serviceManifest)) {
+            executeCreateServicePush(script, config)
+        }
+    }
+}
+
+private def executeCreateServicePush(script, Map config) {
+    dockerExecute(script:script,dockerImage: config.dockerImage, dockerWorkspace: config.dockerWorkspace) {
+
+        String varPart = varOptions(config)
+
+        String varFilePart = varFileOptions(config)
+
+        withCredentials([
+            usernamePassword(credentialsId: config.cloudFoundry.credentialsId, passwordVariable: 'CF_PASSWORD', usernameVariable: 'CF_USERNAME')
+        ]) {
+            def returnCode = sh returnStatus: true, script: """#!/bin/bash
+            set +x
+            set -e
+            export HOME=${config.dockerWorkspace}
+            cf login -u ${BashUtils.quoteAndEscape(CF_USERNAME)} -p ${BashUtils.quoteAndEscape(CF_PASSWORD)} -a ${config.cloudFoundry.apiEndpoint} -o ${BashUtils.quoteAndEscape(config.cloudFoundry.org)} -s ${BashUtils.quoteAndEscape(config.cloudFoundry.space)};
+            cf create-service-push --no-push -f ${BashUtils.quoteAndEscape(config.cloudFoundry.serviceManifest)}${varPart}${varFilePart}
+            """
+            sh "cf logout"
+            if (returnCode!=0)  {
+                error "[${STEP_NAME}] ERROR: The execution of the create-service-push plugin failed, see the logs above for more details."
+            }
+        }
+    }
+}
+
+private varOptions(Map config) {
+    String varPart = ''
+    if (config.cloudFoundry.manifestVariables) {
+        if (!(config.cloudFoundry.manifestVariables in List)) {
+            error "[${STEP_NAME}] ERROR: Parameter config.cloudFoundry.manifestVariables is not a List!"
+        }
+        config.cloudFoundry.manifestVariables.each {
+            if (!(it in Map)) {
+                error "[${STEP_NAME}] ERROR: Parameter config.cloudFoundry.manifestVariables.$it is not a Map!"
+            }
+            it.keySet().each { varKey ->
+                String varValue=BashUtils.quoteAndEscape(it.get(varKey).toString())
+                varPart += " --var $varKey=$varValue"
+            }
+        }
+    }
+    if (varPart) echo "We will add the following string to the cf push call: '$varPart'"
+    return varPart
+}
+
+private String varFileOptions(Map config) {
+    String varFilePart = ''
+    if (config.cloudFoundry.manifestVariablesFiles) {
+        if (!(config.cloudFoundry.manifestVariablesFiles in List)) {
+            error "[${STEP_NAME}] ERROR: Parameter config.cloudFoundry.manifestVariablesFiles is not a List!"
+        }
+        config.cloudFoundry.manifestVariablesFiles.each {
+            if (fileExists(it)) {
+                varFilePart += " --vars-file ${BashUtils.quoteAndEscape(it)}"
+            } else {
+                echo "[${STEP_NAME}] [WARNING] We skip adding not-existing file '$it' as a vars-file to the cf create-service-push call"
+            }
+        }
+    }
+    if (varFilePart) echo "We will add the following string to the cf push call: '$varFilePart'"
+    return varFilePart
+}

vars/cloudFoundryDeploy.groovy

@@ -6,6 +6,7 @@ import com.sap.piper.GenerateDocumentation
 import com.sap.piper.Utils
 import com.sap.piper.ConfigurationHelper
 import com.sap.piper.CfManifestUtils
+import com.sap.piper.BashUtils
 
 import groovy.transform.Field
 
@@ -35,6 +36,35 @@ import groovy.transform.Field
          * @parentConfigKey cloudFoundry
          */
         'manifest',
+        /**
+         * Defines the manifest variables Yaml files to be used to replace variable references in manifest. This parameter
+         * is optional and will default to `["manifest-variables.yml"]`. This can be used to set variable files like it
+         * is provided by `cf push --vars-file <file>`.
+         *
+         * If the manifest is present and so are all variable files, a variable substitution will be triggered that uses
+         * the `cfManifestSubstituteVariables` step before deployment. The format of variable references follows the
+         * [Cloud Foundry standard](https://docs.cloudfoundry.org/devguide/deploy-apps/manifest-attributes.html#variable-substitution).
+         * @parentConfigKey cloudFoundry
+         */
+        'manifestVariablesFiles',
+        /**
+         * Defines a `List` of variables as key-value `Map` objects used for variable substitution within the file given by `manifest`.
+         * Defaults to an empty list, if not specified otherwise. This can be used to set variables like it is provided
+         * by `cf push --var key=value`.
+         *
+         * The order of the maps of variables given in the list is relevant in case there are conflicting variable names and values
+         * between maps contained within the list. In case of conflicts, the last specified map in the list will win.
+         *
+         * Though each map entry in the list can contain more than one key-value pair for variable substitution, it is recommended
+         * to stick to one entry per map, and rather declare more maps within the list. The reason is that
+         * if a map in the list contains more than one key-value entry, and the entries are conflicting, the
+         * conflict resolution behavior is undefined (since map entries have no sequence).
+         *
+         * Note: variables defined via `manifestVariables` always win over conflicting variables defined via any file given
+         * by `manifestVariablesFiles` - no matter what is declared before. This is the same behavior as can be
+         * observed when using `cf push --var` in combination with `cf push --vars-file`.
+         */
+        'manifestVariables',
         /**
          * Cloud Foundry target organization.
          * @parentConfigKey cloudFoundry
@@ -67,7 +97,21 @@ import groovy.transform.Field
     /** @see dockerExecute */
     'stashContent',
     /**
-     * Defines additional parameters passed to mta for deployment with the mtaDeployPlugin.
+     * Additional parameters passed to cf native deployment command.
+     */
+    'cfNativeDeployParameters',
+    /**
+     * Addition command line options for cf api command.
+     * No escaping/quoting is performed. Not recommanded for productive environments.
+     */
+    'apiParameters',
+    /**
+     * Addition command line options for cf login command.
+     * No escaping/quoting is performed. Not recommanded for productive environments.
+     */
+    'loginParameters',
+    /**
+     * Additional parameters passed to mta deployment command.
      */
     'mtaDeployParameters',
     /**
@@ -86,10 +130,15 @@ import groovy.transform.Field
     /**
      * Expected status code returned by the check.
      */
-    'smokeTestStatusCode'
+    'smokeTestStatusCode',
+    /**
+      * Provides more output. May reveal sensitive information.
+      * @possibleValues true, false
+      */
+    'verbose',
 ]
 
-@Field Map CONFIG_KEY_COMPATIBILITY = [cloudFoundry: [apiEndpoint: 'cfApiEndpoint', appName:'cfAppName', credentialsId: 'cfCredentialsId', manifest: 'cfManifest', org: 'cfOrg', space: 'cfSpace']]
+@Field Map CONFIG_KEY_COMPATIBILITY = [cloudFoundry: [apiEndpoint: 'cfApiEndpoint', appName:'cfAppName', credentialsId: 'cfCredentialsId', manifest: 'cfManifest', manifestVariablesFiles: 'cfManifestVariablesFiles', manifestVariables: 'cfManifestVariables',  org: 'cfOrg', space: 'cfSpace']]
 
 @Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS
 
@@ -157,49 +206,26 @@ void call(Map parameters = [:]) {
         //make sure that for further execution whole workspace, e.g. also downloaded artifacts are considered
         config.stashContent = []
 
-        boolean deploy = false
+        boolean deployTriggered = false
         boolean deploySuccess = true
         try {
             if (config.deployTool == 'mtaDeployPlugin') {
-                deploy = true
-                // set default mtar path
-                config = ConfigurationHelper.newInstance(this, config)
-                    .addIfEmpty('mtaPath', config.mtaPath?:findMtar())
-                    .use()
-
-                dockerExecute(script: script, dockerImage: config.dockerImage, dockerWorkspace: config.dockerWorkspace, stashContent: config.stashContent) {
-                    deployMta(config)
-                }
+                deployTriggered = true
+                handleMTADeployment(config, script)
             }
-
-            if (config.deployTool == 'cf_native') {
-                deploy = true
-                config.smokeTest = ''
-
-                if (config.smokeTestScript == 'blueGreenCheckScript.sh') {
-                    writeFile file: config.smokeTestScript, text: libraryResource(config.smokeTestScript)
-                }
-
-                config.smokeTest = '--smoke-test $(pwd)/' + config.smokeTestScript
-                sh "chmod +x ${config.smokeTestScript}"
-
-                echo "[${STEP_NAME}] CF native deployment (${config.deployType}) with cfAppName=${config.cloudFoundry.appName}, cfManifest=${config.cloudFoundry.manifest}, smokeTestScript=${config.smokeTestScript}"
-
-                dockerExecute (
-                    script: script,
-                    dockerImage: config.dockerImage,
-                    dockerWorkspace: config.dockerWorkspace,
-                    stashContent: config.stashContent,
-                    dockerEnvVars: [CF_HOME:"${config.dockerWorkspace}", CF_PLUGIN_HOME:"${config.dockerWorkspace}", STATUS_CODE: "${config.smokeTestStatusCode}"]
-                ) {
-                    deployCfNative(config)
-                }
+            else if (config.deployTool == 'cf_native') {
+                deployTriggered = true
+                handleCFNativeDeployment(config, script)
+            }
+            else {
+                deployTriggered = false
+                echo "[${STEP_NAME}] WARNING! Found unsupported deployTool. Skipping deployment."
             }
         } catch (err) {
             deploySuccess = false
             throw err
         } finally {
-            if (deploy) {
+            if (deployTriggered) {
                 reportToInflux(script, config, deploySuccess, jenkinsUtils)
             }
 
@@ -208,6 +234,17 @@ void call(Map parameters = [:]) {
     }
 }
 
+private void handleMTADeployment(Map config, script) {
+    // set default mtar path
+    config = ConfigurationHelper.newInstance(this, config)
+        .addIfEmpty('mtaPath', config.mtaPath ?: findMtar())
+        .use()
+
+    dockerExecute(script: script, dockerImage: config.dockerImage, dockerWorkspace: config.dockerWorkspace, stashContent: config.stashContent) {
+        deployMta(config)
+    }
+}
+
 def findMtar(){
     def mtarFiles = findFiles(glob: '**/*.mtar')
 
@@ -220,87 +257,6 @@ def findMtar(){
     error 'No *.mtar file found!'
 }
 
-def deployCfNative (config) {
-    withCredentials([usernamePassword(
-        credentialsId: config.cloudFoundry.credentialsId,
-        passwordVariable: 'password',
-        usernameVariable: 'username'
-    )]) {
-        def deployCommand = selectCfDeployCommandForDeployType(config)
-
-        if (config.deployType == 'blue-green') {
-            handleLegacyCfManifest(config)
-        } else {
-            config.smokeTest = ''
-        }
-
-        def blueGreenDeployOptions = deleteOptionIfRequired(config)
-
-        // check if appName is available
-        if (config.cloudFoundry.appName == null || config.cloudFoundry.appName == '') {
-            if (config.deployType == 'blue-green') {
-                error "[${STEP_NAME}] ERROR: Blue-green plugin requires app name to be passed (see https://github.com/bluemixgaragelondon/cf-blue-green-deploy/issues/27)"
-            }
-            if (fileExists(config.cloudFoundry.manifest)) {
-                def manifest = readYaml file: config.cloudFoundry.manifest
-                if (!manifest || !manifest.applications || !manifest.applications[0].name)
-                    error "[${STEP_NAME}] ERROR: No appName available in manifest ${config.cloudFoundry.manifest}."
-
-            } else {
-                error "[${STEP_NAME}] ERROR: No manifest file ${config.cloudFoundry.manifest} found."
-            }
-        }
-
-        def returnCode = sh returnStatus: true, script: """#!/bin/bash
-            set +x
-            set -e
-            export HOME=${config.dockerWorkspace}
-            cf login -u \"${username}\" -p '${password}' -a ${config.cloudFoundry.apiEndpoint} -o \"${config.cloudFoundry.org}\" -s \"${config.cloudFoundry.space}\"
-            cf plugins
-            cf ${deployCommand} ${config.cloudFoundry.appName ?: ''} ${blueGreenDeployOptions} -f '${config.cloudFoundry.manifest}' ${config.smokeTest}
-            """
-        if(returnCode != 0){
-            error "[ERROR][${STEP_NAME}] The execution of the deploy command failed, see the log for details."
-        }
-        stopOldAppIfRunning(config)
-        sh "cf logout"
-    }
-}
-
-private String selectCfDeployCommandForDeployType(Map config) {
-    if (config.deployType == 'blue-green') {
-        return 'blue-green-deploy'
-    } else {
-        return 'push'
-    }
-}
-
-private String deleteOptionIfRequired(Map config) {
-    boolean deleteOldInstance = !config.keepOldInstance
-    if (deleteOldInstance && config.deployType == 'blue-green') {
-        return '--delete-old-apps'
-    } else {
-        return ''
-    }
-}
-
-private void stopOldAppIfRunning(Map config) {
-    String oldAppName = "${config.cloudFoundry.appName}-old"
-    String cfStopOutputFileName = "${UUID.randomUUID()}-cfStopOutput.txt"
-
-    if (config.keepOldInstance && config.deployType == 'blue-green') {
-        int cfStopReturncode = sh (returnStatus: true, script: "cf stop $oldAppName  &> $cfStopOutputFileName")
-
-        if (cfStopReturncode > 0) {
-            String cfStopOutput = readFile(file: cfStopOutputFileName)
-
-            if (!cfStopOutput.contains("$oldAppName not found")) {
-                error "Could not stop application $oldAppName. Error: $cfStopOutput"
-            }
-        }
-    }
-}
-
 def deployMta (config) {
     if (config.mtaExtensionDescriptor == null) config.mtaExtensionDescriptor = ''
     if (!config.mtaExtensionDescriptor.isEmpty() && !config.mtaExtensionDescriptor.startsWith('-e ')) config.mtaExtensionDescriptor = "-e ${config.mtaExtensionDescriptor}"
@@ -313,24 +269,59 @@ def deployMta (config) {
         }
     }
 
-    withCredentials([usernamePassword(
-        credentialsId: config.cloudFoundry.credentialsId,
-        passwordVariable: 'password',
-        usernameVariable: 'username'
-    )]) {
-        echo "[${STEP_NAME}] Deploying MTA (${config.mtaPath}) with following parameters: ${config.mtaExtensionDescriptor} ${config.mtaDeployParameters}"
-        def returnCode = sh returnStatus: true, script: """#!/bin/bash
-            export HOME=${config.dockerWorkspace}
-            set +x
-            set -e
-            cf api ${config.cloudFoundry.apiEndpoint}
-            cf login -u ${username} -p '${password}' -a ${config.cloudFoundry.apiEndpoint} -o \"${config.cloudFoundry.org}\" -s \"${config.cloudFoundry.space}\"
-            cf plugins
-            cf ${deployCommand} ${config.mtaPath} ${config.mtaDeployParameters} ${config.mtaExtensionDescriptor}"""
-        if(returnCode != 0){
-            error "[ERROR][${STEP_NAME}] The execution of the deploy command failed, see the log for details."
-        }
-        sh "cf logout"
+    def deployStatement = "cf ${deployCommand} ${config.mtaPath} ${config.mtaDeployParameters} ${config.mtaExtensionDescriptor}"
+    def apiStatement = "cf api ${config.cloudFoundry.apiEndpoint} ${config.apiParameters}"
+
+    echo "[${STEP_NAME}] Deploying MTA (${config.mtaPath}) with following parameters: ${config.mtaExtensionDescriptor} ${config.mtaDeployParameters}"
+    deploy(apiStatement, deployStatement, config, null)
+}
+
+private void handleCFNativeDeployment(Map config, script) {
+    config.smokeTest = ''
+
+    if (config.deployType == 'blue-green') {
+        prepareBlueGreenCfNativeDeploy(config,script)
+    } else {
+        prepareCfPushCfNativeDeploy(config)
+    }
+
+    echo "[${STEP_NAME}] CF native deployment (${config.deployType}) with:"
+    echo "[${STEP_NAME}] - cfAppName=${config.cloudFoundry.appName}"
+    echo "[${STEP_NAME}] - cfManifest=${config.cloudFoundry.manifest}"
+    echo "[${STEP_NAME}] - cfManifestVariables=${config.cloudFoundry.manifestVariables?:'none specified'}"
+    echo "[${STEP_NAME}] - cfManifestVariablesFiles=${config.cloudFoundry.manifestVariablesFiles?:'none specified'}"
+    echo "[${STEP_NAME}] - smokeTestScript=${config.smokeTestScript}"
+
+    checkIfAppNameIsAvailable(config)
+    dockerExecute(
+        script: script,
+        dockerImage: config.dockerImage,
+        dockerWorkspace: config.dockerWorkspace,
+        stashContent: config.stashContent,
+        dockerEnvVars: [CF_HOME: "${config.dockerWorkspace}", CF_PLUGIN_HOME: "${config.dockerWorkspace}", STATUS_CODE: "${config.smokeTestStatusCode}"]
+    ) {
+        deployCfNative(config)
+    }
+}
+
+private prepareBlueGreenCfNativeDeploy(config,script) {
+    if (config.smokeTestScript == 'blueGreenCheckScript.sh') {
+        writeFile file: config.smokeTestScript, text: libraryResource(config.smokeTestScript)
+    }
+
+    config.smokeTest = '--smoke-test $(pwd)/' + config.smokeTestScript
+    sh "chmod +x ${config.smokeTestScript}"
+
+    config.deployCommand = 'blue-green-deploy'
+    cfManifestSubstituteVariables(
+        script: script,
+        manifestFile: config.cloudFoundry.manifest,
+        manifestVariablesFiles: config.cloudFoundry.manifestVariablesFiles,
+        manifestVariables: config.cloudFoundry.manifestVariables
+    )
+    handleLegacyCfManifest(config)
+    if (!config.keepOldInstance) {
+        config.deployOptions = '--delete-old-apps'
     }
 }
 
@@ -349,6 +340,136 @@ Transformed manifest file content: $transformedManifest"""
     }
 }
 
+private prepareCfPushCfNativeDeploy(config) {
+    config.deployCommand = 'push'
+    config.deployOptions = "${varOptions(config)}${varFileOptions(config)}"
+}
+
+private varOptions(Map config) {
+    String varPart = ''
+    if (config.cloudFoundry.manifestVariables) {
+        if (!(config.cloudFoundry.manifestVariables in List)) {
+            error "[${STEP_NAME}] ERROR: Parameter config.cloudFoundry.manifestVariables is not a List!"
+        }
+        config.cloudFoundry.manifestVariables.each {
+            if (!(it in Map)) {
+                error "[${STEP_NAME}] ERROR: Parameter config.cloudFoundry.manifestVariables.$it is not a Map!"
+            }
+            it.keySet().each { varKey ->
+                String varValue=BashUtils.quoteAndEscape(it.get(varKey).toString())
+                varPart += " --var $varKey=$varValue"
+            }
+        }
+    }
+    if (varPart) echo "We will add the following string to the cf push call:$varPart !"
+    return varPart
+}
+
+private String varFileOptions(Map config) {
+    String varFilePart = ''
+    if (config.cloudFoundry.manifestVariablesFiles) {
+        if (!(config.cloudFoundry.manifestVariablesFiles in List)) {
+            error "[${STEP_NAME}] ERROR: Parameter config.cloudFoundry.manifestVariablesFiles is not a List!"
+        }
+        config.cloudFoundry.manifestVariablesFiles.each {
+            if (fileExists(it)) {
+                varFilePart += " --vars-file ${BashUtils.quoteAndEscape(it)}"
+            } else {
+                echo "[${STEP_NAME}] [WARNING] We skip adding not-existing file '$it' as a vars-file to the cf create-service-push call"
+            }
+        }
+    }
+    if (varFilePart) echo "We will add the following string to the cf push call:$varFilePart !"
+    return varFilePart
+}
+
+private checkIfAppNameIsAvailable(config) {
+    if (config.cloudFoundry.appName == null || config.cloudFoundry.appName == '') {
+        if (config.deployType == 'blue-green') {
+            error "[${STEP_NAME}] ERROR: Blue-green plugin requires app name to be passed (see https://github.com/bluemixgaragelondon/cf-blue-green-deploy/issues/27)"
+        }
+        if (fileExists(config.cloudFoundry.manifest)) {
+            def manifest = readYaml file: config.cloudFoundry.manifest
+            if (!manifest || !manifest.applications || !manifest.applications[0].name) {
+                error "[${STEP_NAME}] ERROR: No appName available in manifest ${config.cloudFoundry.manifest}."
+            }
+        } else {
+            error "[${STEP_NAME}] ERROR: No manifest file ${config.cloudFoundry.manifest} found."
+        }
+    }
+}
+
+def deployCfNative (config) {
+    def deployStatement = "cf ${config.deployCommand} ${config.cloudFoundry.appName ?: ''} ${config.deployOptions?:''} -f '${config.cloudFoundry.manifest}' ${config.smokeTest} ${config.cfNativeDeployParameters}"
+    deploy(null, deployStatement, config,  { c -> stopOldAppIfRunning(c) })
+}
+
+private deploy(def cfApiStatement, def cfDeployStatement, def config, Closure postDeployAction) {
+
+    withCredentials([usernamePassword(
+        credentialsId: config.cloudFoundry.credentialsId,
+        passwordVariable: 'password',
+        usernameVariable: 'username'
+    )]) {
+
+        def cfTraceFile = 'cf.log'
+
+        def deployScript = """#!/bin/bash
+            set +x
+            set -e
+            export HOME=${config.dockerWorkspace}
+            export CF_TRACE=${cfTraceFile}
+            ${cfApiStatement ?: ''}
+            cf login -u \"${username}\" -p '${password}' -a ${config.cloudFoundry.apiEndpoint} -o \"${config.cloudFoundry.org}\" -s \"${config.cloudFoundry.space}\" ${config.loginParameters}
+            cf plugins
+            ${cfDeployStatement}
+            """
+
+        if(config.verbose) {
+            // Password contained in output below is hidden by withCredentials
+            echo "[INFO][${STEP_NAME}] Executing command: '${deployScript}'."
+        }
+
+        def returnCode = sh returnStatus: true, script: deployScript
+
+        if(config.verbose || returnCode != 0) {
+            if(fileExists(file: cfTraceFile)) {
+                echo  '### START OF CF CLI TRACE OUTPUT ###'
+                // Would be nice to inline the two next lines, but that is not understood by the test framework
+                def cfTrace =  readFile(file: cfTraceFile)
+                echo cfTrace
+                echo '### END OF CF CLI TRACE OUTPUT ###'
+            } else {
+                echo "No trace file found at '${cfTraceFile}'"
+            }
+        }
+
+        if(returnCode != 0){
+            error "[${STEP_NAME}] ERROR: The execution of the deploy command failed, see the log for details."
+        }
+
+        if(postDeployAction) postDeployAction(config)
+
+        sh "cf logout"
+    }
+}
+
+private void stopOldAppIfRunning(Map config) {
+    String oldAppName = "${config.cloudFoundry.appName}-old"
+    String cfStopOutputFileName = "${UUID.randomUUID()}-cfStopOutput.txt"
+
+    if (config.keepOldInstance && config.deployType == 'blue-green') {
+        int cfStopReturncode = sh (returnStatus: true, script: "cf stop $oldAppName  &> $cfStopOutputFileName")
+
+        if (cfStopReturncode > 0) {
+            String cfStopOutput = readFile(file: cfStopOutputFileName)
+
+            if (!cfStopOutput.contains("$oldAppName not found")) {
+                error "[${STEP_NAME}] ERROR: Could not stop application $oldAppName. Error: $cfStopOutput"
+            }
+        }
+    }
+}
 
 private void reportToInflux(script, config, deploySuccess, JenkinsUtils jenkinsUtils) {
     def deployUser = ''

vars/commonPipelineEnvironment.groovy

@@ -17,6 +17,8 @@ class commonPipelineEnvironment implements Serializable {
     String gitHttpsUrl
     String gitBranch
 
+    String xsDeploymentId
+
     //GiutHub specific information
     String githubOrg
     String githubRepo

vars/dockerExecute.groovy

@@ -1,3 +1,5 @@
+import com.sap.piper.SidecarUtils
+
 import static com.sap.piper.Prerequisites.checkScript
 
 import com.cloudbees.groovy.cps.NonCPS
@@ -120,10 +122,12 @@ void call(Map parameters = [:], body) {
             .loadStepDefaults()
             .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS)
             .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS)
-            .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName?:env.STAGE_NAME, STEP_CONFIG_KEYS)
+            .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName ?: env.STAGE_NAME, STEP_CONFIG_KEYS)
             .mixin(parameters, PARAMETER_KEYS)
             .use()
 
+        SidecarUtils sidecarUtils = new SidecarUtils(script)
+
         new Utils().pushToSWA([
             step: STEP_NAME,
             stepParamKey1: 'scriptMissing',
@@ -133,16 +137,26 @@ void call(Map parameters = [:], body) {
         ], config)
 
         if (isKubernetes() && config.dockerImage) {
+            List dockerEnvVars = []
+            config.dockerEnvVars?.each { key, value ->
+                dockerEnvVars << "$key=$value"
+            }
             if (env.POD_NAME && isContainerDefined(config)) {
                 container(getContainerDefined(config)) {
-                    echo "[INFO][${STEP_NAME}] Executing inside a Kubernetes Container."
-                    body()
-                    sh "chown -R 1000:1000 ."
+                    withEnv(dockerEnvVars) {
+                        echo "[INFO][${STEP_NAME}] Executing inside a Kubernetes Container."
+                        body()
+                        sh "chown -R 1000:1000 ."
+                    }
                 }
             } else {
+                if (!config.dockerName) {
+                    config.dockerName = UUID.randomUUID().toString()
+                }
                 if (!config.sidecarImage) {
                     dockerExecuteOnKubernetes(
                         script: script,
+                        containerName: config.dockerName,
                         containerCommand: config.containerCommand,
                         containerShell: config.containerShell,
                         dockerImage: config.dockerImage,
@@ -155,43 +169,24 @@ void call(Map parameters = [:], body) {
                         body()
                     }
                 } else {
-                    if(!config.dockerName){
-                        config.dockerName = UUID.randomUUID().toString()
-                    }
-
-                    Map paramMap = [
+                    dockerExecuteOnKubernetes(
                         script: script,
-                        containerCommands: [:],
-                        containerEnvVars: [:],
-                        containerPullImageFlags: [:],
-                        containerMap: [:],
                         containerName: config.dockerName,
-                        containerPortMappings: [:],
-                        containerWorkspaces: [:],
-                        stashContent: config.stashContent
-                    ]
-
-                    paramMap.containerCommands[config.sidecarImage] = ''
-
-                    paramMap.containerEnvVars[config.dockerImage] = config.dockerEnvVars
-                    paramMap.containerEnvVars[config.sidecarImage] = config.sidecarEnvVars
-
-                    paramMap.containerPullImageFlags[config.dockerImage] = config.dockerPullImage
-                    paramMap.containerPullImageFlags[config.sidecarImage] = config.sidecarPullImage
-
-                    paramMap.containerMap[config.dockerImage] = config.dockerName
-                    paramMap.containerMap[config.sidecarImage] = config.sidecarName
-
-                    paramMap.containerPortMappings = config.containerPortMappings
-
-                    paramMap.containerWorkspaces[config.dockerImage] = config.dockerWorkspace
-                    paramMap.containerWorkspaces[config.sidecarImage] = ''
-
-                    dockerExecuteOnKubernetes(paramMap){
-                        echo "[INFO][${STEP_NAME}] Executing inside a Kubernetes Pod with sidecar container"
-                        if(config.sidecarReadyCommand) {
-                            waitForSidecarReadyOnKubernetes(config.sidecarName, config.sidecarReadyCommand)
-                        }
+                        containerCommand: config.containerCommand,
+                        containerShell: config.containerShell,
+                        dockerImage: config.dockerImage,
+                        dockerPullImage: config.dockerPullImage,
+                        dockerEnvVars: config.dockerEnvVars,
+                        dockerWorkspace: config.dockerWorkspace,
+                        stashContent: config.stashContent,
+                        containerPortMappings: config.containerPortMappings,
+                        sidecarName: parameters.sidecarName,
+                        sidecarImage: parameters.sidecarImage,
+                        sidecarPullImage: parameters.sidecarPullImage,
+                        sidecarReadyCommand: parameters.sidecarReadyCommand,
+                        sidecarEnvVars: parameters.sidecarEnvVars
+                    ) {
+                        echo "[INFO][${STEP_NAME}] Executing inside a Kubernetes Pod"
                         body()
                     }
                 }
@@ -212,7 +207,7 @@ void call(Map parameters = [:], body) {
                 utils.unstashAll(config.stashContent)
                 def image = docker.image(config.dockerImage)
                 if (config.dockerPullImage) image.pull()
-                else echo"[INFO][$STEP_NAME] Skipped pull of image '${config.dockerImage}'."
+                else echo "[INFO][$STEP_NAME] Skipped pull of image '${config.dockerImage}'."
                 if (!config.sidecarImage) {
                     image.inside(getDockerOptions(config.dockerEnvVars, config.dockerVolumeBind, config.dockerOptions)) {
                         body()
@@ -220,28 +215,28 @@ void call(Map parameters = [:], body) {
                 } else {
                     def networkName = "sidecar-${UUID.randomUUID()}"
                     sh "docker network create ${networkName}"
-                    try{
+                    try {
                         def sidecarImage = docker.image(config.sidecarImage)
                         if (config.sidecarPullImage) sidecarImage.pull()
-                        else echo"[INFO][$STEP_NAME] Skipped pull of image '${config.sidecarImage}'."
-                        config.sidecarOptions = config.sidecarOptions?:[]
+                        else echo "[INFO][$STEP_NAME] Skipped pull of image '${config.sidecarImage}'."
+                        config.sidecarOptions = config.sidecarOptions ?: []
                         if (config.sidecarName)
                             config.sidecarOptions.add("--network-alias ${config.sidecarName}")
                         config.sidecarOptions.add("--network ${networkName}")
                         sidecarImage.withRun(getDockerOptions(config.sidecarEnvVars, config.sidecarVolumeBind, config.sidecarOptions)) { container ->
-                            config.dockerOptions = config.dockerOptions?:[]
+                            config.dockerOptions = config.dockerOptions ?: []
                             if (config.dockerName)
                                 config.dockerOptions.add("--network-alias ${config.dockerName}")
                             config.dockerOptions.add("--network ${networkName}")
-                            if(config.sidecarReadyCommand) {
-                                waitForSidecarReadyOnDocker(container.id, config.sidecarReadyCommand)
+                            if (config.sidecarReadyCommand) {
+                                sidecarUtils.waitForSidecarReadyOnDocker(container.id, config.sidecarReadyCommand)
                             }
                             image.inside(getDockerOptions(config.dockerEnvVars, config.dockerVolumeBind, config.dockerOptions)) {
                                 echo "[INFO][${STEP_NAME}] Running with sidecar container."
                                 body()
                             }
                         }
-                    }finally{
+                    } finally {
                         sh "docker network remove ${networkName}"
                     }
                 }
@@ -253,41 +248,13 @@ void call(Map parameters = [:], body) {
     }
 }
 
-private waitForSidecarReadyOnDocker(String containerId, String command){
-    String dockerCommand = "docker exec ${containerId} ${command}"
-    waitForSidecarReady(dockerCommand)
-}
-
-private waitForSidecarReadyOnKubernetes(String containerName, String command){
-    container(name: containerName){
-        waitForSidecarReady(command)
-    }
-}
-
-private waitForSidecarReady(String command){
-    int sleepTimeInSeconds = 10
-    int timeoutInSeconds = 5 * 60
-    int maxRetries = timeoutInSeconds / sleepTimeInSeconds
-    int retries = 0
-    while(true){
-        echo "Waiting for sidecar container"
-        String status = sh script:command, returnStatus:true
-        if(status == "0") return
-        if(retries > maxRetries){
-            error("Timeout while waiting for sidecar container to be ready")
-        }
-
-        sleep sleepTimeInSeconds
-        retries++
-    }
-}
-
 /*
  * Returns a string with docker options containing
  * environment variables (if set).
  * Possible to extend with further options.
  * @param dockerEnvVars Map with environment variables
  */
+
 @NonCPS
 private getDockerOptions(Map dockerEnvVars, Map dockerVolumeBind, def dockerOptions) {
     def specialEnvironments = [
@@ -358,14 +325,15 @@ boolean isKubernetes() {
  * E.g. <code>description=Lorem ipsum</code> is
  * changed to <code>description=Lorem\ ipsum</code>.
  */
+
 @NonCPS
 def escapeBlanks(def s) {
 
-    def EQ='='
-    def parts=s.split(EQ)
+    def EQ = '='
+    def parts = s.split(EQ)
 
-    if(parts.length == 2) {
-        parts[1]=parts[1].replaceAll(' ', '\\\\ ')
+    if (parts.length == 2) {
+        parts[1] = parts[1].replaceAll(' ', '\\\\ ')
         s = parts.join(EQ)
     }
 

vars/dockerExecuteOnKubernetes.groovy

@@ -1,3 +1,5 @@
+import com.sap.piper.SidecarUtils
+
 import static com.sap.piper.Prerequisites.checkScript
 
 import com.sap.piper.ConfigurationHelper
@@ -77,6 +79,40 @@ import hudson.AbortException
      * Specifies a dedicated user home directory for the container which will be passed as value for environment variable `HOME`.
      */
     'dockerWorkspace',
+    /**
+     * as `dockerImage` for the sidecar container
+     */
+    'sidecarImage',
+    /**
+     * SideCar only:
+     * Name of the container in local network.
+     */
+    'sidecarName',
+    /**
+     * Set this to 'false' to bypass a docker image pull.
+     * Usefull during development process. Allows testing of images which are available in the local registry only.
+     */
+    'sidecarPullImage',
+    /**
+     * Command executed inside the container which returns exit code 0 when the container is ready to be used.
+     */
+    'sidecarReadyCommand',
+    /**
+     * as `dockerEnvVars` for the sidecar container
+     */
+    'sidecarEnvVars',
+    /**
+     * as `dockerWorkspace` for the sidecar container
+     */
+    'sidecarWorkspace',
+    /**
+     * as `dockerVolumeBind` for the sidecar container
+     */
+    'sidecarVolumeBind',
+    /**
+     * as `dockerOptions` for the sidecar container
+     */
+    'sidecarOptions',
     /** Defines the Kubernetes nodeSelector as per [https://github.com/jenkinsci/kubernetes-plugin](https://github.com/jenkinsci/kubernetes-plugin).*/
     'nodeSelector',
     /**
@@ -90,11 +126,21 @@ import hudson.AbortException
      */
     'stashContent',
     /**
+     * In the Kubernetes case the workspace is only available to the respective Jenkins slave but not to the containers running inside the pod.<br />
+     * This configuration defines exclude pattern for stashing from Jenkins workspace to working directory in container and back.
+     * Following excludes can be set:
      *
+     * * `workspace`: Pattern for stashing towards container
+     * * `stashBack`: Pattern for bringing data from container back to Jenkins workspace. If not set: defaults to setting for `workspace`.
      */
     'stashExcludes',
     /**
+     * In the Kubernetes case the workspace is only available to the respective Jenkins slave but not to the containers running inside the pod.<br />
+     * This configuration defines include pattern for stashing from Jenkins workspace to working directory in container and back.
+     * Following includes can be set:
      *
+     * * `workspace`: Pattern for stashing towards container
+     * * `stashBack`: Pattern for bringing data from container back to Jenkins workspace. If not set: defaults to setting for `workspace`.
      */
     'stashIncludes'
 ])
@@ -138,27 +184,27 @@ void call(Map parameters = [:], body) {
         Map config = configHelper.use()
 
         new Utils().pushToSWA([
-            step: STEP_NAME,
+            step         : STEP_NAME,
             stepParamKey1: 'scriptMissing',
-            stepParam1: parameters?.script == null
+            stepParam1   : parameters?.script == null
         ], config)
 
-        if (!parameters.containerMap) {
+        if (!config.containerMap) {
             configHelper.withMandatoryProperty('dockerImage')
             config.containerName = 'container-exec'
-            config.containerMap = ["${config.get('dockerImage')}": config.containerName]
-            config.containerCommands = config.containerCommand ? ["${config.get('dockerImage')}": config.containerCommand] : null
+            config.containerMap = [(config.get('dockerImage')): config.containerName]
+            config.containerCommands = config.containerCommand ? [(config.get('dockerImage')): config.containerCommand] : null
         }
-        executeOnPod(config, utils, body)
+        executeOnPod(config, utils, body, script)
     }
 }
 
 def getOptions(config) {
     def namespace = config.jenkinsKubernetes.namespace
     def options = [
-        name      : 'dynamic-agent-' + config.uniqueId,
-        label     : config.uniqueId,
-        yaml      : generatePodSpec(config)
+        name : 'dynamic-agent-' + config.uniqueId,
+        label: config.uniqueId,
+        yaml : generatePodSpec(config)
     ]
     if (namespace) {
         options.namespace = namespace
@@ -172,7 +218,7 @@ def getOptions(config) {
     return options
 }
 
-void executeOnPod(Map config, utils, Closure body) {
+void executeOnPod(Map config, utils, Closure body, Script script) {
     /*
      * There could be exceptions thrown by
         - The podTemplate
@@ -184,25 +230,28 @@ void executeOnPod(Map config, utils, Closure body) {
      * In case third case, we need to create the 'container' stash to bring the modified content back to the host.
      */
     try {
-
+        SidecarUtils sidecarUtils = new SidecarUtils(script)
         def stashContent = config.stashContent
-        if (config.containerName && stashContent.isEmpty()){
+        if (config.containerName && stashContent.isEmpty()) {
             stashContent = [stashWorkspace(config, 'workspace')]
         }
         podTemplate(getOptions(config)) {
             node(config.uniqueId) {
+                if (config.sidecarReadyCommand) {
+                    sidecarUtils.waitForSidecarReadyOnKubernetes(config.sidecarName, config.sidecarReadyCommand)
+                }
                 if (config.containerName) {
                     Map containerParams = [name: config.containerName]
                     if (config.containerShell) {
                         containerParams.shell = config.containerShell
                     }
                     echo "ContainerConfig: ${containerParams}"
-                    container(containerParams){
+                    container(containerParams) {
                         try {
                             utils.unstashAll(stashContent)
                             body()
                         } finally {
-                            stashWorkspace(config, 'container', true)
+                            stashWorkspace(config, 'container', true, true)
                         }
                     }
                 } else {
@@ -220,11 +269,11 @@ private String generatePodSpec(Map config) {
     def containers = getContainerList(config)
     def podSpec = [
         apiVersion: "v1",
-        kind: "Pod",
-        metadata: [
+        kind      : "Pod",
+        metadata  : [
             lables: config.uniqueId
         ],
-        spec: [
+        spec      : [
             containers: containers
         ]
     ]
@@ -234,23 +283,34 @@ private String generatePodSpec(Map config) {
 }
 
 
-private String stashWorkspace(config, prefix, boolean chown = false) {
+private String stashWorkspace(config, prefix, boolean chown = false, boolean stashBack = false) {
     def stashName = "${prefix}-${config.uniqueId}"
     try {
-        if (chown)  {
+        if (chown) {
             def securityContext = getSecurityContext(config)
             def runAsUser = securityContext?.runAsUser ?: 1000
             def fsGroup = securityContext?.fsGroup ?: 1000
-            sh """#!${config.containerShell?:'/bin/sh'}
+            sh """#!${config.containerShell ?: '/bin/sh'}
 chown -R ${runAsUser}:${fsGroup} ."""
         }
+
+        def includes, excludes
+
+        if (stashBack) {
+            includes = config.stashIncludes.stashBack ?: config.stashIncludes.workspace
+            excludes = config.stashExcludes.stashBack ?: config.stashExcludes.workspace
+        } else {
+            includes = config.stashIncludes.workspace
+            excludes = config.stashExcludes.workspace
+        }
+
         stash(
             name: stashName,
-            includes: config.stashIncludes.workspace,
-            excludes: config.stashExcludes.workspace,
-            //inactive due to negative side-effects, we may require a dedicated git stash to be used
-            //useDefaultExcludes: false
+            includes: includes,
+            excludes: excludes
         )
+        //inactive due to negative side-effects, we may require a dedicated git stash to be used
+        //useDefaultExcludes: false)
         return stashName
     } catch (AbortException | IOException e) {
         echo "${e.getMessage()}"
@@ -274,21 +334,21 @@ private List getContainerList(config) {
 
     //If no custom jnlp agent provided as default jnlp agent (jenkins/jnlp-slave) as defined in the plugin, see https://github.com/jenkinsci/kubernetes-plugin#pipeline-support
     def result = []
-
     //allow definition of jnlp image via environment variable JENKINS_JNLP_IMAGE in the Kubernetes landscape or via config as fallback
     if (env.JENKINS_JNLP_IMAGE || config.jenkinsKubernetes.jnlpAgent) {
         result.push([
-            name: 'jnlp',
+            name : 'jnlp',
             image: env.JENKINS_JNLP_IMAGE ?: config.jenkinsKubernetes.jnlpAgent
         ])
     }
     config.containerMap.each { imageName, containerName ->
         def containerPullImage = config.containerPullImageFlags?.get(imageName)
+        boolean pullImage = containerPullImage != null ? containerPullImage : config.dockerPullImage
         def containerSpec = [
-            name: containerName.toLowerCase(),
-            image: imageName,
-            imagePullPolicy: containerPullImage ? "Always" : "IfNotPresent",
-            env: getContainerEnvs(config, imageName)
+            name           : containerName.toLowerCase(),
+            image          : imageName,
+            imagePullPolicy: pullImage ? "Always" : "IfNotPresent",
+            env            : getContainerEnvs(config, imageName)
         ]
 
         def configuredCommand = config.containerCommands?.get(imageName)
@@ -299,32 +359,43 @@ private List getContainerList(config) {
                 '-f',
                 '/dev/null'
             ]
-        } else if(configuredCommand != "") {
+        } else if (configuredCommand != "") {
             // apparently "" is used as a flag for not settings container commands !?
             containerSpec['command'] =
-                    (configuredCommand in List) ? configuredCommand : [
-                        shell,
-                        '-c',
-                        configuredCommand
-                    ]
+                (configuredCommand in List) ? configuredCommand : [
+                    shell,
+                    '-c',
+                    configuredCommand
+                ]
         }
 
         if (config.containerPortMappings?.get(imageName)) {
             def ports = []
             def portCounter = 0
-            config.containerPortMappings.get(imageName).each {mapping ->
+            config.containerPortMappings.get(imageName).each { mapping ->
                 def name = "${containerName}${portCounter}".toString()
-                if(mapping.containerPort != mapping.hostPort) {
-                    echo ("[WARNING][${STEP_NAME}]: containerPort and hostPort are different for container '${containerName}'. "
+                if (mapping.containerPort != mapping.hostPort) {
+                    echo("[WARNING][${STEP_NAME}]: containerPort and hostPort are different for container '${containerName}'. "
                         + "The hostPort will be ignored.")
                 }
                 ports.add([name: name, containerPort: mapping.containerPort])
-                portCounter ++
+                portCounter++
             }
             containerSpec.ports = ports
         }
         result.push(containerSpec)
     }
+    if (config.sidecarImage) {
+        def containerSpec = [
+            name           : config.sidecarName.toLowerCase(),
+            image          : config.sidecarImage,
+            imagePullPolicy: config.sidecarPullImage ? "Always" : "IfNotPresent",
+            env            : getContainerEnvs(config, config.sidecarImage),
+            command        : []
+        ]
+
+        result.push(containerSpec)
+    }
     return result
 }
 
@@ -334,13 +405,14 @@ private List getContainerList(config) {
  * (Kubernetes-Plugin only!)
  * @param config Map with configurations
  */
+
 private List getContainerEnvs(config, imageName) {
     def containerEnv = []
     def dockerEnvVars = config.containerEnvVars?.get(imageName) ?: config.dockerEnvVars ?: [:]
     def dockerWorkspace = config.containerWorkspaces?.get(imageName) != null ? config.containerWorkspaces?.get(imageName) : config.dockerWorkspace ?: ''
 
     def envVar = { e ->
-        [ name: e.key, value: e.value ]
+        [name: e.key, value: e.value]
     }
 
     if (dockerEnvVars) {

vars/gaugeExecuteTests.groovy

@@ -5,7 +5,7 @@ import com.sap.piper.Utils
 import com.sap.piper.ConfigurationHelper
 import com.sap.piper.GitUtils
 import com.sap.piper.analytics.InfluxData
-import groovy.text.SimpleTemplateEngine
+import groovy.text.GStringTemplateEngine
 import groovy.transform.Field
 
 @Field String STEP_NAME = getClass().getName()