Merge remote-tracking branch 'github/master' into HEAD

This commit is contained in:
Marcus Holl
2019-03-21 10:43:27 +01:00
11 changed files with 273 additions and 144 deletions
+1 -1
View File
@@ -10,7 +10,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.sap.cp.jenkins</groupId>
<artifactId>jenkins-library</artifactId>
<version>0.9</version>
<version>0.10</version>
<name>SAP CP Piper Library</name>
<description>Shared library containing steps and utilities to set up continuous deployment processes for SAP technologies.</description>
+4 -4
View File
@@ -13,9 +13,9 @@ ${error}
***
Further information:
* Documentation of library step ${stepName}: https://sap.github.io/jenkins-library/steps/${stepName}/
* Source code of library step ${stepName}: https://github.com/SAP/jenkins-library/blob/master/vars/${stepName}.groovy
* Library documentation: https://sap.github.io/jenkins-library/
* Library repository: https://github.com/SAP/jenkins-library
* Documentation of library step ${stepName}: ${libraryDocumentationUrl}steps/${stepName}/
* Source code of library step ${stepName}: ${libraryRepositoryUrl}blob/master/vars/${stepName}.groovy
* Library documentation: ${libraryDocumentationUrl}
* Library repository: ${libraryRepositoryUrl}
----------------------------------------------------------
@@ -39,6 +39,10 @@ general:
gitSshKeyCredentialsId: '' #needed to allow sshagent to run with local ssh key
jenkinsKubernetes:
jnlpAgent: 's4sdk/jenkins-agent-k8s:latest'
securityContext:
# Setting security context globally is currently not working with jaas
# runAsUser: 1000
# fsGroup: 1000
manualConfirmation: true
productiveBranch: 'master'
@@ -215,6 +219,10 @@ steps:
languageRunner: 'js'
runCommand: 'gauge run'
testOptions: 'specs'
handlePipelineStepErrors:
echoDetails: true
libraryDocumentationUrl: 'https://sap.github.io/jenkins-library/'
libraryRepositoryUrl: 'https://github.com/SAP/jenkins-library/'
healthExecuteCheck:
healthEndpoint: ''
influxWriteData:
+2
View File
@@ -20,6 +20,7 @@ import org.junit.rules.RuleChain
import groovy.io.FileType
import hudson.AbortException
import util.BasePiperTest
import util.JenkinsReadYamlRule
import util.JenkinsStepRule
import util.Rules
@@ -30,6 +31,7 @@ public class CommonStepsTest extends BasePiperTest{
@Rule
public RuleChain ruleChain = Rules.getCommonRules(this)
.around(new JenkinsReadYamlRule(this))
/*
* With that test we ensure the very first action inside a method body of a call method
@@ -7,6 +7,7 @@ import org.junit.rules.ExpectedException
import org.junit.rules.RuleChain
import groovy.json.JsonSlurper
import util.BasePiperTest
import util.JenkinsDockerExecuteRule
import util.JenkinsLoggingRule
@@ -54,7 +55,8 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
def portList = []
def containerCommands = []
def pullImageMap = [:]
def namespace
def securityContext
@Before
void init() {
@@ -71,21 +73,25 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
helper.registerAllowedMethod('podTemplate', [Map.class, Closure.class], { Map options, Closure body ->
podName = options.name
podLabel = options.label
options.containers.each { option ->
containersList.add(option.name)
imageList.add(option.image.toString())
envList.add(option.envVars)
portList.add(option.ports)
if (option.command) {
containerCommands.add(option.command)
namespace = options.namespace
def podSpec = new JsonSlurper().parseText(options.yaml) // this yaml is actually json
def containers = podSpec.spec.containers
securityContext = podSpec.spec.securityContext
containers.each { container ->
containersList.add(container.name)
imageList.add(container.image.toString())
envList.add(container.env)
if(container.ports) {
portList.add(container.ports)
}
pullImageMap.put(option.image.toString(), option.alwaysPullImage)
if (container.command) {
containerCommands.add(container.command)
}
pullImageMap.put(container.image.toString(), container.imagePullPolicy == "Always")
}
body()
})
helper.registerAllowedMethod('node', [String.class, Closure.class], { String nodeName, Closure body -> body() })
helper.registerAllowedMethod('envVar', [Map.class], { Map option -> return option })
helper.registerAllowedMethod('containerTemplate', [Map.class], { Map option -> return option })
}
@Test
@@ -257,10 +263,10 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
hasItem('maven:3.5-jdk-8-alpine'),
hasItem('selenium/standalone-chrome'),
))
assertThat(portList, hasItem(hasItem([name: 'selenium0', containerPort: 4444, hostPort: 4444])))
assertThat(portMapping, hasItem([name: 'selenium0', containerPort: 4444, hostPort: 4444]))
// assertThat(portList, is(null))
assertThat(portList, hasItem([[name: 'selenium0', containerPort: 4444, hostPort: 4444]]))
assertThat(containerCommands.size(), is(1))
assertThat(envList, hasItem(hasItem(allOf(hasEntry('key', 'customEnvKey'), hasEntry ('value','customEnvValue')))))
assertThat(envList, hasItem(hasItem(allOf(hasEntry('name', 'customEnvKey'), hasEntry ('value','customEnvValue')))))
}
@Test
@@ -286,7 +292,7 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
) {
//nothing to exeute
}
assertThat(containerCommands, hasItem('/busybox/tail -f /dev/null'))
assertThat(containerCommands, hasItem(['/bin/sh', '-c', '/busybox/tail -f /dev/null']))
}
@Test
@@ -334,6 +340,36 @@ class DockerExecuteOnKubernetesTest extends BasePiperTest {
assertTrue(bodyExecuted)
}
@Test
void testDockerExecuteOnKubernetesWithCustomNamespace() {
def expectedNamespace = "sandbox"
nullScript.commonPipelineEnvironment.configuration = [general: [jenkinsKubernetes: [namespace: expectedNamespace]]]
stepRule.step.dockerExecuteOnKubernetes(
script: nullScript,
juStabUtils: utils,
dockerImage: 'maven:3.5-jdk-8-alpine',
) { bodyExecuted = true }
assertTrue(bodyExecuted)
assertThat(namespace, is(equalTo(expectedNamespace)))
}
@Test
void testDockerExecuteOnKubernetesWithSecurityContext() {
def expectedSecurityContext = [ runAsUser: 1000, fsGroup: 1000 ]
nullScript.commonPipelineEnvironment.configuration = [general: [jenkinsKubernetes: [
securityContext: expectedSecurityContext]]]
stepRule.step.dockerExecuteOnKubernetes(
script: nullScript,
juStabUtils: utils,
dockerImage: 'maven:3.5-jdk-8-alpine',
) { bodyExecuted = true }
assertTrue(bodyExecuted)
assertThat(securityContext, is(equalTo(expectedSecurityContext)))
}
private container(options, body) {
containerName = options.name
containerShell = options.shell
@@ -11,6 +11,7 @@ import static org.junit.Assert.assertThat
import util.BasePiperTest
import util.JenkinsLoggingRule
import util.JenkinsReadYamlRule
import util.JenkinsStepRule
import util.Rules
@@ -22,6 +23,7 @@ class HandlePipelineStepErrorsTest extends BasePiperTest {
@Rule
public RuleChain rules = Rules
.getCommonRules(this)
.around(new JenkinsReadYamlRule(this))
.around(loggingRule)
.around(stepRule)
.around(thrown)
+50 -59
View File
@@ -1,5 +1,11 @@
import com.sap.piper.Utils
import hudson.AbortException
import static org.hamcrest.Matchers.allOf
import static org.hamcrest.Matchers.containsString
import static org.hamcrest.Matchers.not
import org.hamcrest.Matchers
import org.jenkinsci.plugins.credentialsbinding.impl.CredentialNotFoundException
import org.junit.Assert
import org.junit.Before
@@ -196,16 +202,6 @@ class NeoDeployTest extends BasePiperTest {
)
}
@Test
void archiveNotProvidedTest() {
thrown.expect(Exception)
thrown.expectMessage('ERROR - NO VALUE AVAILABLE FOR source')
stepRule.step.neoDeploy(script: nullScript)
}
@Test
void wrongArchivePathProvidedTest() {
@@ -217,14 +213,55 @@ class NeoDeployTest extends BasePiperTest {
@Test
void scriptNotProvidedTest() {
void sanityChecksDeployModeMTATest() {
thrown.expect(Exception)
thrown.expectMessage('ERROR - NO VALUE AVAILABLE FOR neo/host')
thrown.expectMessage(
allOf(
containsString('ERROR - NO VALUE AVAILABLE FOR:'),
containsString('neo/host'),
containsString('neo/account'),
containsString('source')))
nullScript.commonPipelineEnvironment.configuration = [:]
stepRule.step.neoDeploy(script: nullScript, source: archiveName)
// deployMode mta is the default, but for the sake of transparency it is better to repeat it.
stepRule.step.neoDeploy(script: nullScript, deployMode: 'mta')
}
@Test
public void sanityChecksDeployModeWarPropertiesFileTest() {
thrown.expect(IllegalArgumentException)
// using this deploy mode 'account' and 'host' are provided by the properties file
thrown.expectMessage(
allOf(
containsString('ERROR - NO VALUE AVAILABLE FOR source'),
not(containsString('neo/host')),
not(containsString('neo/account'))))
nullScript.commonPipelineEnvironment.configuration = [:]
stepRule.step.neoDeploy(script: nullScript, deployMode: 'warPropertiesFile')
}
@Test
public void sanityChecksDeployModeWarParamsTest() {
thrown.expect(IllegalArgumentException)
thrown.expectMessage(
allOf(
containsString('ERROR - NO VALUE AVAILABLE FOR:'),
containsString('source'),
containsString('neo/application'),
containsString('neo/runtime'),
containsString('neo/runtimeVersion'),
containsString('neo/host'),
containsString('neo/account')))
nullScript.commonPipelineEnvironment.configuration = [:]
stepRule.step.neoDeploy(script: nullScript, deployMode: 'warParams')
}
@Test
@@ -414,52 +451,6 @@ class NeoDeployTest extends BasePiperTest {
.hasSingleQuotedOption('source', '.*\\.war'))
}
@Test
void applicationNameNotProvidedTest() {
thrown.expect(Exception)
thrown.expectMessage('ERROR - NO VALUE AVAILABLE FOR neo/application')
stepRule.step.neoDeploy(script: nullScript,
source: warArchiveName,
deployMode: 'warParams',
neo: [
runtime: 'neo-javaee6-wp',
runtimeVersion: '2.125'
]
)
}
@Test
void runtimeNotProvidedTest() {
thrown.expect(Exception)
thrown.expectMessage('ERROR - NO VALUE AVAILABLE FOR neo/runtime')
stepRule.step.neoDeploy(script: nullScript,
source: warArchiveName,
neo: [
application: 'testApp',
runtimeVersion: '2.125'
],
deployMode: 'warParams')
}
@Test
void runtimeVersionNotProvidedTest() {
thrown.expect(Exception)
thrown.expectMessage('ERROR - NO VALUE AVAILABLE FOR neo/runtimeVersion')
stepRule.step.neoDeploy(script: nullScript,
source: warArchiveName,
neo: [
application: 'testApp',
runtime: 'neo-javaee6-wp'
],
deployMode: 'warParams')
}
@Test
void illegalDeployModeTest() {
+80 -21
View File
@@ -5,6 +5,8 @@ import com.sap.piper.GenerateDocumentation
import com.sap.piper.JenkinsUtils
import com.sap.piper.Utils
import com.sap.piper.k8s.SystemEnv
import com.sap.piper.JsonUtils
import groovy.transform.Field
import hudson.AbortException
@@ -81,7 +83,13 @@ import hudson.AbortException
/**
*
*/
'stashIncludes'
'stashIncludes',
/**
* Kubernetes Security Context used for the pod.
* Can be used to specify uid and fsGroup.
* See: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
*/
'securityContext'
])
@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS.minus([
'stashIncludes',
@@ -124,9 +132,16 @@ void call(Map parameters = [:], body) {
}
def getOptions(config) {
return [name : 'dynamic-agent-' + config.uniqueId,
label : config.uniqueId,
containers: getContainerList(config)]
def namespace = config.jenkinsKubernetes.namespace
def options = [
name : 'dynamic-agent-' + config.uniqueId,
label : config.uniqueId,
yaml : generatePodSpec(config)
]
if (namespace) {
options.namespace = namespace
}
return options
}
void executeOnPod(Map config, utils, Closure body) {
@@ -171,13 +186,33 @@ void executeOnPod(Map config, utils, Closure body) {
}
}
private String generatePodSpec(Map config) {
def containers = getContainerList(config)
def podSpec = [
apiVersion: "v1",
kind: "Pod",
metadata: [
lables: config.uniqueId
],
spec: [
containers: containers
]
]
podSpec.spec.securityContext = getSecurityContext(config)
return new JsonUtils().getPrettyJsonString(podSpec)
}
private String stashWorkspace(config, prefix, boolean chown = false) {
def stashName = "${prefix}-${config.uniqueId}"
try {
// Every dockerImage used in the dockerExecuteOnKubernetes should have user id 1000
if (chown) {
def securityContext = getSecurityContext(config)
def runAsUser = securityContext?.runAsUser ?: 1000
def fsGroup = securityContext?.fsGroup ?: 1000
sh """#!${config.containerShell?:'/bin/sh'}
chown -R 1000:1000 ."""
chown -R ${runAsUser}:${fsGroup} ."""
}
stash(
name: stashName,
@@ -191,6 +226,10 @@ chown -R 1000:1000 ."""
return null
}
private Map getSecurityContext(Map config) {
return config.securityContext ?: config.jenkinsKubernetes.securityContext ?: [:]
}
private void unstashWorkspace(config, prefix) {
try {
unstash "${prefix}-${config.uniqueId}"
@@ -200,25 +239,46 @@ private void unstashWorkspace(config, prefix) {
}
private List getContainerList(config) {
result = []
result.push(containerTemplate(
def result = [[
name: 'jnlp',
image: config.jenkinsKubernetes.jnlpAgent
))
]]
config.containerMap.each { imageName, containerName ->
def containerPullImage = config.containerPullImageFlags?.get(imageName)
def templateParameters = [
def containerSpec = [
name: containerName.toLowerCase(),
image: imageName,
alwaysPullImage: containerPullImage != null ? containerPullImage : config.dockerPullImage,
envVars: getContainerEnvs(config, imageName)
imagePullPolicy: containerPullImage ? "Always" : "IfNotPresent",
env: getContainerEnvs(config, imageName)
]
if (!config.containerCommands?.get(imageName)?.isEmpty()) {
templateParameters.command = config.containerCommands?.get(imageName)?: '/usr/bin/tail -f /dev/null'
def configuredCommand = config.containerCommands?.get(imageName)
def shell = config.containerShell ?: '/bin/sh'
if (configuredCommand == null) {
containerSpec['command'] = [
'/usr/bin/tail',
'-f',
'/dev/null'
]
} else if(configuredCommand != "") {
// apparently "" is used as a flag for not settings container commands !?
containerSpec['command'] =
(configuredCommand in List) ? configuredCommand : [
shell,
'-c',
configuredCommand
]
}
if (config.containerPortMappings?.get(imageName)) {
def portMapping = { m ->
[
name: m.name,
containerPort: m.containerPort,
hostPort: m.hostPort
]
}
def ports = []
def portCounter = 0
config.containerPortMappings.get(imageName).each {mapping ->
@@ -226,9 +286,9 @@ private List getContainerList(config) {
ports.add(portMapping(mapping))
portCounter ++
}
templateParameters.ports = ports
containerSpec.ports = ports
}
result.push(containerTemplate(templateParameters))
result.push(containerSpec)
}
return result
}
@@ -244,6 +304,10 @@ private List getContainerEnvs(config, imageName) {
def dockerEnvVars = config.containerEnvVars?.get(imageName) ?: config.dockerEnvVars ?: [:]
def dockerWorkspace = config.containerWorkspaces?.get(imageName) != null ? config.containerWorkspaces?.get(imageName) : config.dockerWorkspace ?: ''
def envVar = { e ->
[ name: e.key, value: e.value ]
}
if (dockerEnvVars) {
for (String k : dockerEnvVars.keySet()) {
containerEnv << envVar(key: k, value: dockerEnvVars[k].toString())
@@ -260,10 +324,5 @@ private List getContainerEnvs(config, imageName) {
containerEnv << envVar(key: env, value: systemEnv.get(env))
}
// ContainerEnv array can't be empty. Using a stub to avoid failure.
if (!containerEnv) {
containerEnv << envVar(key: "EMPTY_VAR", value: " ")
}
return containerEnv
}
+48 -20
View File
@@ -1,40 +1,68 @@
import com.cloudbees.groovy.cps.NonCPS
import com.sap.piper.ConfigurationHelper
import groovy.text.SimpleTemplateEngine
import groovy.transform.Field
@Field STEP_NAME = getClass().getName()
@Field Set GENERAL_CONFIG_KEYS = []
@Field Set STEP_CONFIG_KEYS = []
@Field Set PARAMETER_KEYS = [
'echoDetails',
'libraryDocumentationUrl',
'libraryRepositoryUrl',
'stepName',
'stepNameDoc',
'stepParameters'
]
void call(Map parameters = [:], body) {
def stepParameters = parameters.stepParameters //mandatory
def stepName = parameters.stepName //mandatory
def verbose = parameters.get('echoDetails', true)
// load default & individual configuration
Map config = ConfigurationHelper.newInstance(this)
.loadStepDefaults()
.mixin(parameters, PARAMETER_KEYS)
.withMandatoryProperty('stepParameters')
.withMandatoryProperty('stepName')
.addIfEmpty('stepNameDoc' , parameters.stepName)
.use()
def message = ''
try {
if (stepParameters == null && stepName == null)
error "The step handlePipelineStepErrors requires following mandatory parameters: stepParameters, stepName"
if (verbose)
echo "--- Begin library step of: ${stepName} ---"
if (config.echoDetails)
echo "--- Begin library step of: ${config.stepName} ---"
body()
} catch (Throwable error) {
if (verbose)
message += SimpleTemplateEngine.newInstance()
.createTemplate(libraryResource('com.sap.piper/templates/error.log'))
.make([
stepName: stepName,
stepParameters: stepParameters?.toString(),
error: error
]).toString()
writeErrorToInfluxData(parameters, error)
if (config.echoDetails)
message += formatErrorMessage(config, error)
writeErrorToInfluxData(config, error)
throw error
} finally {
if (verbose)
message += "--- End library step of: ${stepName} ---"
if (config.echoDetails)
message += "--- End library step of: ${config.stepName} ---"
echo message
}
}
private void writeErrorToInfluxData(config, error){
@NonCPS
private String formatErrorMessage(Map config, error){
Map binding = [
error: error,
libraryDocumentationUrl: config.libraryDocumentationUrl,
libraryRepositoryUrl: config.libraryRepositoryUrl,
stepName: config.stepName,
stepParameters: config.stepParameters?.toString()
]
return SimpleTemplateEngine
.newInstance()
.createTemplate(libraryResource('com.sap.piper/templates/error.log'))
.make(binding)
.toString()
}
private void writeErrorToInfluxData(Map config, error){
def script = config?.stepParameters?.script
if(script && script.commonPipelineEnvironment?.getInfluxCustomDataMapTags().build_error_message == null){
+11 -6
View File
@@ -41,23 +41,28 @@ void call(parameters = [:]) {
.mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName ?: env.STAGE_NAME, STEP_CONFIG_KEYS)
.addIfEmpty('source', script.commonPipelineEnvironment.getMtarFilePath())
.mixin(parameters, PARAMETER_KEYS)
.withMandatoryProperty('neo/host')
.withMandatoryProperty('neo/account')
.withMandatoryProperty('source')
.withMandatoryProperty('neo/credentialsId')
.collectValidationFailures()
.withPropertyInValues('deployMode', DeployMode.stringValues())
Map configuration = configHelper.use()
DeployMode deployMode = DeployMode.fromString(configuration.deployMode)
def isWarParamsDeployMode = { deployMode == DeployMode.WAR_PARAMS }
def isWarParamsDeployMode = { deployMode == DeployMode.WAR_PARAMS },
isNotWarPropertiesDeployMode = {deployMode != DeployMode.WAR_PROPERTIES_FILE}
configHelper
.withMandatoryProperty('source')
.withMandatoryProperty('neo/credentialsId')
.withMandatoryProperty('neo/application', null, isWarParamsDeployMode)
.withMandatoryProperty('neo/runtime', null, isWarParamsDeployMode)
.withMandatoryProperty('neo/runtimeVersion', null, isWarParamsDeployMode)
.withMandatoryProperty('neo/host', null, isNotWarPropertiesDeployMode)
.withMandatoryProperty('neo/account', null, isNotWarPropertiesDeployMode)
//
// call 'use()' a second time in order to get the collected validation failures
// since the map did not change, it is not required to replace the previous configuration map.
.use()
utils.pushToSWA([
step: STEP_NAME,
+15 -17
View File
@@ -6,24 +6,22 @@ import groovy.transform.Field
@Field STEP_NAME = getClass().getName()
void call(Map parameters = [:]) {
handlePipelineStepErrors (stepName: 'prepareDefaultValues', stepParameters: parameters, echoDetails: false) {
if(!DefaultValueCache.getInstance() || parameters.customDefaults) {
def defaultValues = [:]
def configFileList = ['default_pipeline_environment.yml']
def customDefaults = parameters.customDefaults
if(!DefaultValueCache.getInstance() || parameters.customDefaults) {
def defaultValues = [:]
def configFileList = ['default_pipeline_environment.yml']
def customDefaults = parameters.customDefaults
if(customDefaults in String)
customDefaults = [customDefaults]
if(customDefaults in List)
configFileList += customDefaults
for (def configFileName : configFileList){
if(configFileList.size() > 1) echo "Loading configuration file '${configFileName}'"
def configuration = readYaml text: libraryResource(configFileName)
defaultValues = MapUtils.merge(
MapUtils.pruneNulls(defaultValues),
MapUtils.pruneNulls(configuration))
}
DefaultValueCache.createInstance(defaultValues)
if(customDefaults in String)
customDefaults = [customDefaults]
if(customDefaults in List)
configFileList += customDefaults
for (def configFileName : configFileList){
if(configFileList.size() > 1) echo "Loading configuration file '${configFileName}'"
def configuration = readYaml text: libraryResource(configFileName)
defaultValues = MapUtils.merge(
MapUtils.pruneNulls(defaultValues),
MapUtils.pruneNulls(configuration))
}
DefaultValueCache.createInstance(defaultValues)
}
}