jenkins/sap-jenkins-library
1
0
Fork 0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-03-05 15:15:44 +02:00
Code Issues Releases Activity

Merge remote-tracking branch 'github/master' into HEAD

Browse Source
This commit is contained in:
Marcus Holl 2019-03-19 09:06:15 +01:00
commit 4d4b9de660
18 changed files with 234 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/CONTRIBUTING.md vendored
View File

@ -24,6 +24,9 @@ Generally speaking, you should fork this repository, make changes in your own fo
All pipeline library coding _must_ come with automated unit tests.
Besides that, we have an integration test suite, which is not triggered during normal pull request builds. However, integration tests are mandatory before a change can be merged. It is the duty of a team member of the SAP/jenkins-library project to execute these tests.
To trigger the integration test suite, the `HEAD` commit of the branch associated with the pull request must be pushed under the branch pattern `it/.*` (recommended naming convention: `it/<Number of the pull request>`). As a result, the status `integration-tests` is updated in the pull request.
### Documentation
The contract of functionality exposed by a library functionality needs to be documented, so it can be properly used.

1
consumer-test/integrationTestController.sh
View File

@ -96,6 +96,7 @@ for p in "${processes[@]}"
do
area=$(dirname "${p%:*}")
testCase=$(basename "${p%:*}")
TEST_CASE_ROOT="${WORKSPACES_ROOT}/${area}/${testCase}"
echo "[INFO] === START === Logs for test case \"${testCase}\" ===."
cat "${TEST_CASE_ROOT}/log.txt"
echo "[INFO] === END === Logs for test case \"${testCase}\" ===."

66
documentation/bin/createDocu.groovy
View File

@ -1,41 +1,15 @@
import groovy.io.FileType;
import groovy.io.FileType
import org.yaml.snakeyaml.Yaml
import org.codehaus.groovy.control.CompilerConfiguration
import com.sap.piper.GenerateDocumentation
import com.sap.piper.DefaultValueCache
import java.util.regex.Matcher
import groovy.text.StreamingTemplateEngine
//
// Collects helper functions for rendering the docu
//
class TemplateHelper {
static replaceParagraph(def textIn, int level, name, replacement) {
boolean insideParagraph = false
def textOut = ''
textIn.eachLine {
line ->
if(insideParagraph && line ==~ "^#{1,${level}} .*\$") {
insideParagraph = false
}
if(! insideParagraph) {
textOut += "${line}\n"
}
if(line ==~ "^#{${level}} ${name}.*\$") {
insideParagraph = true
textOut += "${replacement}\n\n"
}
}
textOut
}
static createParametersTable(Map parameters) {
def t = ''
@ -61,18 +35,22 @@ class TemplateHelper {
t.trim()
}
static createParametersSection(Map parameters) {
createParametersTable(parameters) + '\n' + createParameterDescriptionSection(parameters)
}
static createStepConfigurationSection(Map parameters) {
def t = '''|We recommend to define values of step parameters via [config.yml file](../configuration.md).
|
|In following sections of the config.yml the configuration is possible:\n\n'''.stripMargin()
t += '| parameter | general | step | stage |\n'
t += '|-----------|---------|------|-------|\n'
t += '| parameter | general | step/stage |\n'
t += '|-----------|---------|------------|\n'
parameters.keySet().toSorted().each {
def props = parameters.get(it)
t += "| `${it}` | ${props.GENERAL_CONFIG ? 'X' : ''} | ${props.STEP_CONFIG ? 'X' : ''} | ${props.STAGE_CONFIG ? 'X' : ''} |\n"
t += "| `${it}` | ${props.GENERAL_CONFIG ? 'X' : ''} | ${props.STEP_CONFIG ? 'X' : ''} |\n"
}
t.trim()
@ -332,7 +310,7 @@ class Helper {
stepsDir.traverse(type: FileType.FILES, maxDepth: 0) {
if(it.getName().endsWith('.groovy')) {
def scriptName = (it =~ /vars\/(.*)\.groovy/)[0][1]
def scriptName = (it =~ /vars\${File.separator}(.*)\.groovy/)[0][1]
def stepScript = gse.createScript("${scriptName}.groovy", new Binding())
for (def method in stepScript.getClass().getMethods()) {
if(method.getName() == 'call' && method.getAnnotation(GenerateDocumentation) != null) {
@ -457,20 +435,15 @@ void renderStep(stepName, stepProperties) {
return
}
def text = theStepDocu.text
if(stepProperties.description) {
text = TemplateHelper.replaceParagraph(text, 2, 'Description', '\n' + stepProperties.description)
}
if(stepProperties.parameters) {
def binding = [
docGenStepName : stepName,
docGenDescription : 'Description\n\n' + stepProperties.description,
docGenParameters : 'Parameters\n\n' + TemplateHelper.createParametersSection(stepProperties.parameters),
docGenConfiguration : 'Step configuration\n\n' + TemplateHelper.createStepConfigurationSection(stepProperties.parameters)
]
def template = new StreamingTemplateEngine().createTemplate(theStepDocu.text)
String text = template.make(binding)
text = TemplateHelper.replaceParagraph(text, 2, 'Parameters', '\n' +
TemplateHelper.createParametersTable(stepProperties.parameters) + '\n' +
TemplateHelper.createParameterDescriptionSection(stepProperties.parameters))
text = TemplateHelper.replaceParagraph(text, 2, 'Step configuration', '\n' +
TemplateHelper.createStepConfigurationSection(stepProperties.parameters))
}
theStepDocu.withWriter { w -> w.write text }
}
@ -544,8 +517,7 @@ def handleStep(stepName, prepareDefaultValuesStep, gse) {
required: true,
GENERAL_CONFIG: false,
STEP_CONFIG: false,
STAGE_CONFIG: false
STEP_CONFIG: false
]
// END special handling for 'script' parameter

14
documentation/docs/steps/checkChangeInDevelopment.md
View File

@ -1,20 +1,14 @@
# checkChangeInDevelopment
# ${docGenStepName}
## Description
Content here is generated from corresponding step, see `vars`.
## ${docGenDescription}
## Prerequisites
* **[Change Management Client 2.0.0 or compatible version](http://central.maven.org/maven2/com/sap/devops/cmclient/dist.cli/)** - available for download on Maven Central.
## Parameters
## ${docGenParameters}
Content here is generated from corresponding step, see `vars`.
## Step configuration
Content here is generated from corresponding step, see `vars`.
## ${docGenConfiguration}
## Exceptions

14
documentation/docs/steps/dockerExecute.md
View File

@ -1,20 +1,14 @@
# dockerExecute
# ${docGenStepName}
## Description
## ${docGenDescription}
Content here is generated from corresponding step, see `vars`.
## Parameters
Content here is generated from corresponding step, see `vars`.
## ${docGenParameters}
## Kubernetes support
If the Jenkins is setup on a Kubernetes cluster, then you can execute the closure inside a container of a pod by setting an environment variable `ON_K8S` to `true`. However, it will ignore `containerPortMappings`, `dockerOptions` and `dockerVolumeBind` values.
## Step configuration
Content here is generated from corresponding step, see `vars`.
## ${docGenConfiguration}
## Side effects

14
documentation/docs/steps/dockerExecuteOnKubernetes.md
View File

@ -1,8 +1,6 @@
# dockerExecuteOnKubernetes
# ${docGenStepName}
## Description
Content here is generated from corresponding step, see `vars`.
## ${docGenDescription}
## Prerequisites
@ -11,13 +9,9 @@ Content here is generated from corresponding step, see `vars`.
![Jenkins environment variable configuration](../images/k8s_env.png)
## Parameters
## ${docGenParameters}
Content here is generated from corresponding step, see `vars`.
## Step configuration
Content here is generated from corresponding step, see `vars`.
## ${docGenConfiguration}
## Side effects

14
documentation/docs/steps/karmaExecuteTests.md
View File

@ -1,21 +1,15 @@
# karmaExecuteTests
# ${docGenStepName}
## Description
Content here is generated from corresponnding step, see `vars`.
## ${docGenDescription}
## Prerequisites
* **running Karma tests** - have a NPM module with running tests executed with Karma
* **configured WebDriver** - have the [`karma-webdriver-launcher`](https://github.com/karma-runner/karma-webdriver-launcher) package installed and a custom, WebDriver-based browser configured in Karma
## Parameters
## ${docGenParameters}
Content here is generated from corresponnding step, see `vars`.
## Step configuration
Content here is generated from corresponnding step, see `vars`.
## ${docGenConfiguration}
## Side effects

7
documentation/docs/steps/neoDeploy.md
View File

@ -16,9 +16,9 @@ Note that a version is formed by `major.minor.patch`, and a version is compatibl
![Jenkins credentials configuration](../images/neo_credentials.png)
* **Neo Java Web SDK 3.39.10 or compatible version** - can be downloaded from [Maven Central](http://central.maven.org/maven2/com/sap/cloud/neo-java-web-sdk/). The Neo Java Web SDK needs to be extracted into the folder provided by `neoHome`. In case this parameters is not provided and there is no NEO_HOME parameter in the environment `<neoRoot>/tools` needs to be in the `PATH`. This step is also capable of triggering the neo deploy tool provided inside a docker image.
* **Neo Java Web SDK 3.39.10 or compatible version** - can be downloaded from [Maven Central](http://central.maven.org/maven2/com/sap/cloud/neo-java-web-sdk/). This step is capable of triggering the neo deploy tool provided inside a docker image. We provide docker image `ppiper/neo-cli`. `neo.sh` needs to be contained in path, e.g by adding a symbolic link to `/usr/local/bin`.
* **Java 8 or compatible version** - needed by the *Neo-Java-Web-SDK*
* **Java 8 or compatible version** - needed by the *Neo-Java-Web-SDK*. Java environment needs to be properly configured (JAVA_HOME, java exectutable contained in path).
## Parameters when using MTA deployment method (default - MTA)
@ -104,12 +104,13 @@ The step is prepared for being executed in docker. The corresponding parameters
## Step configuration
The parameter `neo` including all can also be specified as a global parameter using the global configuration file.
The parameter `neo` including all options can also be specified as a global parameter using the global configuration file.
The following parameters can also be specified as step parameters using the global configuration file:
* `dockerImage`
* `neoHome`
* `source`
## Side effects

12
documentation/docs/steps/npmExecute.md
View File

@ -1,16 +1,12 @@
# npmExecute
## Description
# ${docGenStepName}
Content here is generated from corresponding step, see `vars`.
## Parameters
## ${docGenDescription}
Content here is generated from corresponding step, see `vars`.
## ${docGenParameters}
## Step configuration
Content here is generated from corresponding step, see `vars`.
## ${docGenConfiguration}
## Exceptions

26
documentation/docs/steps/snykExecute.md Normal file
View File

@ -0,0 +1,26 @@
# ${docGenStepName}
## ${docGenDescription}
## Prerequisites
* **Snyk account** - have an account on snyk.io
* **Snyk token** - have a Snyk user token
## ${docGenParameters}
## ${docGenConfiguration}
## Side effects
Step uses `dockerExecute` inside.
## Exceptions
none
## Example
```groovy
snykExecute script: this, snykCredentialsId: 'mySnykToken'
```

14
documentation/docs/steps/uiVeri5ExecuteTests.md
View File

@ -1,18 +1,12 @@
# uiVeri5ExecuteTests
# ${docGenStepName}
## Description
Content here is generated from corresponding step, see `vars`.
## ${docGenDescription}
## Prerequisites
## Parameters
## ${docGenParameters}
Content here is generated from corresponding step, see `vars`.
## Step configuration
Content here is generated from corresponding step, see `vars`.
## ${docGenConfiguration}
## Exceptions

1
documentation/mkdocs.yml
View File

@ -32,6 +32,7 @@ nav:
- seleniumExecuteTests: steps/seleniumExecuteTests.md
- setupCommonPipelineEnvironment: steps/setupCommonPipelineEnvironment.md
- slackSendNotification: steps/slackSendNotification.md
- snykExecute: steps/snykExecute.md
- testsPublishResults: steps/testsPublishResults.md
- toolValidate: steps/toolValidate.md
- transportRequestCreate: steps/transportRequestCreate.md

19
test/groovy/CheckChangeInDevelopmentTest.groovy
View File

@ -4,7 +4,7 @@ import org.junit.Test
import org.junit.rules.ExpectedException
import org.junit.rules.RuleChain
import com.sap.piper.GitUtils
import com.sap.piper.cm.BackendType
import com.sap.piper.cm.ChangeManagement
import com.sap.piper.cm.ChangeManagementException
@ -155,6 +155,23 @@ class CheckChangeInDevelopmentTest extends BasePiperTest {
}
@Test
public void stageConfigIsNotConsideredWithParamKeysTest() {
nullScript.commonPipelineEnvironment.configuration = [stages:[foo:[changeDocumentId:'12345']]]
ChangeManagement cm = getChangeManagementUtils(true, '')
thrown.expect(IllegalArgumentException)
thrown.expectMessage('No changeDocumentId provided.')
stepRule.step.checkChangeInDevelopment(
script: nullScript,
cmUtils: cm,
changeManagement: [type: BackendType.SOLMAN,
endpoint: 'https://example.org/cm'],
stageName: 'foo')
}
private ChangeManagement getChangeManagementUtils(boolean inDevelopment, String changeDocumentId = '001') {
return new ChangeManagement(nullScript, null) {

47
test/groovy/DockerExecuteTest.groovy
View File

@ -9,6 +9,7 @@ import org.junit.rules.RuleChain
import util.BasePiperTest
import util.JenkinsLoggingRule
import util.JenkinsReadYamlRule
import util.JenkinsShellCallRule
import util.JenkinsStepRule
import util.PluginMock
import util.Rules
@ -23,6 +24,7 @@ class DockerExecuteTest extends BasePiperTest {
private DockerMock docker
private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this)
private JenkinsStepRule stepRule = new JenkinsStepRule(this)
private JenkinsShellCallRule shellRule = new JenkinsShellCallRule(this)
@Rule
public RuleChain ruleChain = Rules
@ -30,8 +32,8 @@ class DockerExecuteTest extends BasePiperTest {
.around(new JenkinsReadYamlRule(this))
.around(loggingRule)
.around(stepRule)
.around(shellRule)
int dockerPsReturnValue = 0
def bodyExecuted
def containerName
@ -41,7 +43,7 @@ class DockerExecuteTest extends BasePiperTest {
docker = new DockerMock()
JenkinsUtils.metaClass.static.isPluginActive = {def s -> new PluginMock(s).isActive()}
binding.setVariable('docker', docker)
helper.registerAllowedMethod('sh', [Map.class], {return dockerPsReturnValue})
shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "docker .*", 0)
}
@Test
@ -201,7 +203,7 @@ class DockerExecuteTest extends BasePiperTest {
@Test
void testDockerNotInstalledResultsInLocalExecution() throws Exception {
dockerPsReturnValue = 1
shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "docker .*", 1)
stepRule.step.dockerExecute(script: nullScript,
dockerOptions: '-it') {
bodyExecuted = true
@ -241,6 +243,18 @@ class DockerExecuteTest extends BasePiperTest {
))
}
@Test
void testSidecarHealthCheck(){
stepRule.step.dockerExecute(
script: nullScript,
dockerImage: 'maven:3.5-jdk-8-alpine',
sidecarImage: 'selenium/standalone-chrome',
sidecarName: 'testAlias',
sidecarReadyCommand: "isReady.sh"
) {}
assertThat(shellRule.shell, hasItem("docker exec uniqueId isReady.sh"))
}
@Test
void testSidecarKubernetes(){
boolean dockerExecuteOnKubernetesCalled = false
@ -275,6 +289,33 @@ class DockerExecuteTest extends BasePiperTest {
assertThat(dockerExecuteOnKubernetesCalled, is(true))
}
@Test
void testSidecarKubernetesHealthCheck(){
binding.setVariable('env', [ON_K8S: 'true'])
helper.registerAllowedMethod('dockerExecuteOnKubernetes', [Map.class, Closure.class], { params, body ->
body()
})
def containerCalled = false
helper.registerAllowedMethod('container', [Map.class, Closure.class], { params, body ->
containerCalled = true
assertThat(params.name, is('testAlias'))
body()
})
stepRule.step.dockerExecute(
script: nullScript,
dockerImage: 'maven:3.5-jdk-8-alpine',
sidecarImage: 'selenium/standalone-chrome',
sidecarName: 'testAlias',
sidecarReadyCommand: "isReady.sh"
) {}
assertThat(containerCalled, is(true))
assertThat(shellRule.shell, hasItem("isReady.sh"))
}
private class DockerMock {
private String imageName
private boolean imagePulled = false

21
test/groovy/NeoDeployTest.groovy
View File

@ -89,7 +89,7 @@ class NeoDeployTest extends BasePiperTest {
}
@Test
void straightForwardTestConfigViaConfiguration() {
void straightForwardTestConfigViaParameters() {
boolean notifyOldConfigFrameworkUsed = true
@ -118,16 +118,19 @@ class NeoDeployTest extends BasePiperTest {
}
@Test
void straightForwardTestConfigViaConfigurationAndViaConfigProperties() {
void straightForwardTestConfigViaConfiguration() {
nullScript.commonPipelineEnvironment.setConfigProperty('DEPLOY_HOST', 'configProperties.deploy.host.com')
nullScript.commonPipelineEnvironment.setConfigProperty('CI_DEPLOY_ACCOUNT', 'configPropsUser123')
nullScript.commonPipelineEnvironment.configuration = [steps: [neoDeploy: [neo: [host : 'configuration-frwk.deploy.host.com',
account: 'configurationFrwkUser123']]]]
nullScript.commonPipelineEnvironment.configuration = [steps: [
neoDeploy: [
neo: [
host: 'configuration-frwk.deploy.host.com',
account: 'configurationFrwkUser123'
],
source: archiveName
]
]]
stepRule.step.neoDeploy(script: nullScript,
source: archiveName,
neo:[credentialsId: 'myCredentialsId']
)
@ -138,7 +141,7 @@ class NeoDeployTest extends BasePiperTest {
.hasOption('synchronous', '')
.hasSingleQuotedOption('user', 'anonymous')
.hasSingleQuotedOption('password', '\\*\\*\\*\\*\\*\\*\\*\\*')
.hasSingleQuotedOption('source', '.*'))
.hasSingleQuotedOption('source', archiveName))
}
@Test

44
vars/dockerExecute.groovy
View File

@ -92,6 +92,10 @@ import groovy.transform.Field
* as `dockerWorkspace` for the sidecar container
*/
'sidecarWorkspace',
/**
* Command executed inside the container which returns exit code 0 when the container is ready to be used.
*/
'sidecarReadyCommand',
/**
* Specific stashes that should be considered for the step execution.
*/
@ -142,6 +146,10 @@ void call(Map parameters = [:], body) {
body()
}
} else {
if(!config.dockerName){
config.dockerName = UUID.randomUUID().toString()
}
Map paramMap = [
script: script,
containerCommands: [:],
@ -153,6 +161,7 @@ void call(Map parameters = [:], body) {
containerWorkspaces: [:],
stashContent: config.stashContent
]
paramMap.containerCommands[config.sidecarImage] = ''
paramMap.containerEnvVars[config.dockerImage] = config.dockerEnvVars
@ -171,6 +180,9 @@ void call(Map parameters = [:], body) {
dockerExecuteOnKubernetes(paramMap){
echo "[INFO][${STEP_NAME}] Executing inside a Kubernetes Pod with sidecar container"
if(config.sidecarReadyCommand) {
waitForSidecarReadyOnKubernetes(config.sidecarName, config.sidecarReadyCommand)
}
body()
}
}
@ -207,11 +219,14 @@ void call(Map parameters = [:], body) {
if (config.sidecarName)
config.sidecarOptions.add("--network-alias ${config.sidecarName}")
config.sidecarOptions.add("--network ${networkName}")
sidecarImage.withRun(getDockerOptions(config.sidecarEnvVars, config.sidecarVolumeBind, config.sidecarOptions)) { c ->
sidecarImage.withRun(getDockerOptions(config.sidecarEnvVars, config.sidecarVolumeBind, config.sidecarOptions)) { container ->
config.dockerOptions = config.dockerOptions?:[]
if (config.dockerName)
config.dockerOptions.add("--network-alias ${config.dockerName}")
config.dockerOptions.add("--network ${networkName}")
if(config.sidecarReadyCommand) {
waitForSidecarReadyOnDocker(container.id, config.sidecarReadyCommand)
}
image.inside(getDockerOptions(config.dockerEnvVars, config.dockerVolumeBind, config.dockerOptions)) {
echo "[INFO][${STEP_NAME}] Running with sidecar container."
body()
@ -229,7 +244,34 @@ void call(Map parameters = [:], body) {
}
}
private waitForSidecarReadyOnDocker(String containerId, String command){
String dockerCommand = "docker exec ${containerId} ${command}"
waitForSidecarReady(dockerCommand)
}
private waitForSidecarReadyOnKubernetes(String containerName, String command){
container(name: containerName){
waitForSidecarReady(command)
}
}
private waitForSidecarReady(String command){
int sleepTimeInSeconds = 10
int timeoutInSeconds = 5 * 60
int maxRetries = timeoutInSeconds / sleepTimeInSeconds
int retries = 0
while(true){
echo "Waiting for sidecar container"
String status = sh script:command, returnStatus:true
if(status == "0") return
if(retries > maxRetries){
error("Timeout while waiting for sidecar container to be ready")
}
sleep sleepTimeInSeconds
retries++
}
}
/*
* Returns a string with docker options containing

4
vars/neoDeploy.groovy
View File

@ -15,11 +15,11 @@ import static com.sap.piper.Prerequisites.checkScript
'dockerEnvVars',
'dockerImage',
'dockerOptions',
'neoHome'
'neoHome',
'source'
])
@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS.plus([
'source',
'deployMode',
'warAction'
])

38
vars/snykExecute.groovy
View File

@ -1,6 +1,7 @@
import static com.sap.piper.Prerequisites.checkScript
import com.sap.piper.ConfigurationHelper
import com.sap.piper.GenerateDocumentation
import com.sap.piper.Utils
import com.sap.piper.mta.MtaMultiplexer
@ -8,19 +9,54 @@ import groovy.transform.Field
@Field def STEP_NAME = getClass().getName()
@Field Set GENERAL_CONFIG_KEYS = ['snykCredentialsId']
@Field Set GENERAL_CONFIG_KEYS = [
/**
* Credentials for accessing the Snyk API.
* @possibleValues Jenkins credentials id
*/
'snykCredentialsId'
]
@Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS.plus([
/**
* The path to the build descriptor file, e.g. `./package.json`.
*/
'buildDescriptorFile',
/** @see dockerExecute */
'dockerImage',
/**
* Only scanType 'mta': Exclude modules from MTA projects.
*/
'exclude',
/**
* Monitor the application's dependencies for new vulnerabilities.
*/
'monitor',
//TODO: move to general
/**
* The type of project that should be scanned.
* @possibleValues `npm`, `mta`
*/
'scanType',
/**
* Only needed for `monitor: true`: The organisation ID to determine the organisation to report to.
*/
'snykOrg',
/**
* Generate and archive a JSON report.
*/
'toJson',
/**
* Generate and archive a HTML report.
*/
'toHtml'
])
@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS
//https://snyk.io/docs/continuous-integration/
/**
* This step performs an open source vulnerability scan on a *Node project* or *Node module inside an MTA project* through snyk.io.
*/
@GenerateDocumentation
void call(Map parameters = [:]) {
handlePipelineStepErrors(stepName: STEP_NAME, stepParameters: parameters) {
def utils = parameters.juStabUtils ?: new Utils()