1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-01-18 05:18:24 +02:00

Merge pull request #617 from marcusholl/pr/resolvePlugins

Resolve plugins
This commit is contained in:
Marcus Holl 2019-05-24 16:53:11 +02:00 committed by GitHub
commit a8648a48ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 430 additions and 11 deletions

View File

@ -1,5 +1,6 @@
import groovy.io.FileType
import groovy.json.JsonOutput
import groovy.json.JsonSlurper
import org.yaml.snakeyaml.Yaml
import org.codehaus.groovy.control.CompilerConfiguration
import com.sap.piper.GenerateDocumentation
@ -14,6 +15,42 @@ import com.sap.piper.MapUtils
//
class TemplateHelper {
static createDependencyList(Set deps) {
def t = ''
t += 'The step depends on the following Jenkins plugins\n\n'
def filteredDeps = deps.findAll { dep -> dep != 'UNIDENTIFIED' }
if(filteredDeps.contains('kubernetes')) {
// The docker plugin is not detected by the tests since it is not
// handled via step call, but it is added to the environment.
// Hovever kubernetes plugin and docker plugin are closely related,
// hence adding docker if kubernetes is present.
filteredDeps.add('docker')
}
if(filteredDeps.isEmpty()) {
t += '* <none>\n'
} else {
filteredDeps
.sort()
.each { dep -> t += "* [${dep}](https://plugins.jenkins.io/${dep})\n" }
}
if(filteredDeps.contains('kubernetes')) {
t += "\nThe kubernetes plugin is only used if running in a kubernetes environment."
}
t += '''|
|Transitive dependencies are omitted.
|
|The list might be incomplete.
|
|Consider using the [ppiper/jenkins-master](https://cloud.docker.com/u/ppiper/repository/docker/ppiper/jenkins-master)
|docker image. This images comes with preinstalled plugins.
|'''.stripMargin()
return t
}
static createParametersTable(Map parameters) {
def t = ''
@ -741,8 +778,10 @@ void renderStep(stepName, stepProperties) {
docGenStepName : stepName,
docGenDescription : 'Description\n\n' + stepProperties.description,
docGenParameters : 'Parameters\n\n' + TemplateHelper.createParametersSection(stepProperties.parameters),
docGenConfiguration : 'Step configuration\n\n' + TemplateHelper.createStepConfigurationSection(stepProperties.parameters)
docGenConfiguration : 'Step configuration\n\n' + TemplateHelper.createStepConfigurationSection(stepProperties.parameters),
docJenkinsPluginDependencies : 'Dependencies\n\n' + TemplateHelper.createDependencyList(stepProperties.dependencies)
]
def template = new StreamingTemplateEngine().createTemplate(theStepDocu.text)
String text = template.make(binding)
@ -802,6 +841,7 @@ def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) {
File theStep = new File(stepsDir, "${stepName}.groovy")
File theStepDocu = new File(stepsDocuDir, "${stepName}.md")
File theStepDeps = new File('documentation/jenkins_workspace/plugin_mapping.json')
if (!theStepDocu.exists() && stepName.indexOf('Stage') != -1) {
//try to get a corresponding stage documentation
@ -859,7 +899,18 @@ def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) {
// 'dependentConfig' is only present here for internal reasons and that entry is removed at
// end of method.
def step = [parameters:[:], dependentConfig: [:]]
def step = [
parameters:[:],
dependencies: (Set)[],
dependentConfig: [:]
]
//
// provide dependencies to Jenkins plugins
if(theStepDeps.exists()) {
def pluginDependencies = new JsonSlurper().parse(theStepDeps)
step.dependencies.addAll(pluginDependencies[stepName].collect { k, v -> k })
}
//
// START special handling for 'script' parameter

View File

@ -3,6 +3,44 @@
d=$(dirname "$0")
[ ! -z "$d" ] && d="$d/"
WS_OUT="$(pwd)/documentation/jenkins_workspace"
WS_IN=/workspace
STEP_CALL_MAPPING_FILE_NAME=step_calls_mapping.json
PLUGIN_MAPPING_FILE_NAME=plugin_mapping.json
CALLS="${WS_OUT}/${STEP_CALL_MAPPING_FILE_NAME}"
PLUGIN_MAPPING="${WS_OUT}/${PLUGIN_MAPPING_FILE_NAME}"
for f in ${CALLS} ${PLUGIN_MAPPING}
do
[ -e "${f}" ] && rm -rf "${f}"
done
export CLASSPATH_FILE='target/cp.txt'
mvn compile dependency:build-classpath -Dmdep.outputFile=${CLASSPATH_FILE} > /dev/null 2>&1
mvn clean test dependency:build-classpath -Dmdep.outputFile=${CLASSPATH_FILE} > /dev/null 2>&1
# --in: is created by the unit tests. It contains a mapping between the test case (name is
# already adjusted).
# --out: Contains a transformed version. The calls to other pipeline steps are resolved in a
# transitive manner. This allows us to report all Jenkins plugin calls (also the calls which
# are performed by other pipeline steps. E.g.: each step includes basically a call to
# handlePipelineStepErrors. The Plugin calls issues by handlePipelineStepErrors are also
# reported for the step calling that auxiliar step).
groovy "${d}resolveTransitiveCalls" -in target/trackedCalls.json --out "${CALLS}"
[ -f "${CALLS}" ] || { echo "File \"${CALLS}\" does not exist." ; exit 1; }
docker run \
-w "${WS_IN}" \
--env calls="${WS_IN}/${STEP_CALL_MAPPING_FILE_NAME}" \
--env result="${WS_IN}/${PLUGIN_MAPPING_FILE_NAME}" \
-v "${WS_OUT}:${WS_IN}" \
ppiper/jenkinsfile-runner \
-ns \
-f Jenkinsfile \
--runWorkspace /workspace
[ -f "${PLUGIN_MAPPING}" ] || { echo "Result file containing step to plugin mapping not found (${PLUGIN_MAPPING})."; exit 1; }
groovy -cp "target/classes:$(cat $CLASSPATH_FILE)" "${d}createDocu" "${@}"

View File

@ -0,0 +1,181 @@
import groovy.json.JsonSlurper
def cli = new CliBuilder(
usage: 'groovy createDocu [<options>]',
header: 'Options:',
footer: 'Copyright: SAP SE')
cli.with {
i longOpt: 'in', args: 1, argName: 'file', 'The file containing the mapping as created by the unit tests..'
o longOpt: 'out', args: 1, argName: 'file', 'The file containing the condenced mappings.'
h longOpt: 'help', 'Prints this help.'
}
def options = cli.parse(args)
if(options.h) {
System.err << "Printing help.\n"
cli.usage()
return
}
if(! options.i) {
System.err << "No input file"
cli.usage()
return
}
if(! options.o) {
System.err << "No output file"
cli.usage()
return
}
def steps = new JsonSlurper().parseText(new File(options.i).text)
def piperSteps = steps.piperSteps
def calls = steps.calls
// only temporary in order to avoid manipulating the map during
// iterating over it.
def tmpCalls = [:]
// Adjust naming
calls.each { c ->
tmpCalls.put(retrieveStepName(c.key), c.value as Set)
}
calls = tmpCalls
tmpCalls = null
// Remove selfs
calls.each { c ->
c.value.remove(c.key)
}
int counter=0
def alreadyHandled = []
//
// in case we exceed the value we assume some cyclic call
// between plugin steps.
int MAX_LOOP = 1600
boolean done = false
while(counter < MAX_LOOP) {
def hereWeNeedToReplace = null
def toBeReplaced = null
if(alreadyHandled.size() == calls.size()) {
done = true
break
}
for (def call in calls.entrySet()) {
stepName = call.key
calledSteps = call.value
if(alreadyHandled.contains(stepName)) {
continue
}
for (def calledStep in calledSteps) {
if(! ( calledStep in Map)) {
// in case the calledStep is a map the map
// was introduced in an earlier loop.
// This means this entry is already handled.
if(calledStep in piperSteps) {
toBeReplaced = calledStep
hereWeNeedToReplace = calledSteps
break
}
}
}
if(toBeReplaced) {
def replacement = [:]
replacement[toBeReplaced] = calls[toBeReplaced] as Set
def removed = hereWeNeedToReplace.remove(toBeReplaced)
hereWeNeedToReplace.add(replacement)
counter++
} else {
alreadyHandled << stepName
}
break
}
}
if(! done) {
throw new Exception('Unable to resolve transitive plugin calls.')
}
piperStepCallMappings = [:]
for(def entry : calls.entrySet()) {
def performedCalls = flatten(entry, (Set)[])
piperStepCallMappings.put(entry.key, performedCalls)
}
//
// special handling since since changeManagement util class
// is separated from the steps itself
//
// should be improved in the future in order not to have
// that bells and whistles here.
def cm = piperStepCallMappings.get('changeManagement')
for (cmStepName in [
'checkChangeInDevelopment',
'transportRequestCreate',
'transportRequestUploadFile',
'transportRequestRelease',
]) {
piperStepCallMappings.get(cmStepName).addAll(cm)
}
// end of special handling
//
File performedCalls = new File(options.o)
if (performedCalls.exists()) performedCalls.delete()
performedCalls << groovy.json.JsonOutput.toJson(piperStepCallMappings)
def flatten(def entry, Set result) {
for(def e : entry.value) {
if(e in Map) { // the map here is expected to hold one entry always
for(def steps : e.entrySet().value) {
for(def step : steps) {
if (step in Map) {
flatten(step, result)
} else {
result << step
}
}
}
} else {
result << e.value.toString()
}
}
result
}
static retrieveStepName(String s) {
firstCharToLowerCase(removeTrailing(s, 'Test'))
}
static removeTrailing(String s, String trail) {
return s.replaceAll(trail + '$', '')
}
static firstCharToLowerCase(CharSequence cs) {
char[] c = cs.getChars()
c[0] = Character.toLowerCase(c[0])
new String(c)
}

View File

@ -10,6 +10,8 @@ none
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Example
```groovy

View File

@ -10,6 +10,8 @@ You need to have a Bats test file. By default you would put this into directory
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Example
```groovy

View File

@ -10,6 +10,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Exceptions
* `AbortException`:

View File

@ -82,6 +82,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
### Thresholds
It is possible to define thresholds to fail the build on a certain count of findings. To achive this, just define your thresholds a followed for the specific check tool:

View File

@ -13,6 +13,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Example
```groovy

View File

@ -10,6 +10,8 @@ Test configuration is available.
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Example
```

View File

@ -9,6 +9,8 @@ You need to store the API token for the Detect service as _'Secret text'_ creden
!!! note "minimum plugin requirement"
This step requires [synopsys-detect-plugin](https://github.com/jenkinsci/synopsys-detect-plugin) with at least version `2.0.0`.
## ${docJenkinsPluginDependencies}
## Example
```groovy

View File

@ -10,6 +10,8 @@ If the Jenkins is setup on a Kubernetes cluster, then you can execute the closur
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Side effects
none

View File

@ -13,6 +13,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Side effects
none

View File

@ -6,6 +6,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Example
```groovy

View File

@ -12,6 +12,8 @@ none
We recommend to define values of step parameters via [config.yml file](../configuration.md).
## ${docJenkinsPluginDependencies}
## Example
Pipeline step:

View File

@ -1,17 +1,19 @@
# ${docGenStepName}
## ${docGenDescription}
## Prerequisites
You need to create a personal access token within GitHub and add this to the Jenkins credentials store.
Please see [GitHub documentation for details about creating the personal access token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/).
## ${docJenkinsPluginDependencies}
## ${docGenParameters}
## ${docGenConfiguration}
## ${docGenDescription}
## Example
Usage of pipeline step:

View File

@ -10,6 +10,8 @@ none
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Example
```groovy

View File

@ -16,6 +16,8 @@ Endpoint for health check is configured.
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Example
Pipeline step:

View File

@ -67,6 +67,8 @@ influxDBServer=jenkins
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Example
```groovy

View File

@ -16,6 +16,8 @@ via _Jenkins_ -> _Credentials_ -> _System_ -> _Global credentials (unrestricted)
* File: upload your `config.json` file
* ID: specify id which you then use for the configuration of `dockerConfigJsonCredentialsId` (see below)
## ${docJenkinsPluginDependencies}
## Example
```groovy

View File

@ -7,6 +7,8 @@
* **running Karma tests** - have a NPM module with running tests executed with Karma
* **configured WebDriver** - have the [`karma-webdriver-launcher`](https://github.com/karma-runner/karma-webdriver-launcher) package installed and a custom, WebDriver-based browser configured in Karma
## ${docJenkinsPluginDependencies}
## ${docGenParameters}
## ${docGenConfiguration}

View File

@ -18,6 +18,8 @@ mailSendNotification script: this
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Side effects
none

View File

@ -6,6 +6,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Exceptions
None

View File

@ -14,6 +14,8 @@ While using a custom docker file, ensure that the following tools are installed:
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Side effects
1. The file name of the resulting archive is written to the `commonPipelineEnvironment` with variable name `mtarFileName`.

View File

@ -1,11 +1,11 @@
# ${docGenStepName}
## ${docGenDescription}
## ${docGenParameters}
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Examples
```groovy

View File

@ -18,6 +18,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Side effects
none

View File

@ -14,6 +14,8 @@
Step uses `dockerExecute` inside.
## ${docJenkinsPluginDependencies}
## Exceptions
none

View File

@ -1,12 +1,12 @@
# ${docGenStepName}
## ${docGenDescription}
## ${docGenParameters}
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Exceptions
None

View File

@ -10,6 +10,8 @@ none
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Side effects
none

View File

@ -31,6 +31,8 @@ pipelineRestartSteps (script: this) {
none
## ${docJenkinsPluginDependencies}
## Exceptions
none

View File

@ -34,6 +34,8 @@ The step is stashing files before and after the build. This is due to the fact,
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Explanation of pipeline step
Usage of pipeline step:

View File

@ -1,7 +1,5 @@
# ${docGenStepName}
## ${docGenDescription}
## Prerequsites
none
@ -9,3 +7,5 @@ none
## ${docGenParameters}
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}

View File

@ -9,3 +9,5 @@ none
## ${docGenParameters}
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}

View File

@ -5,3 +5,5 @@
## ${docGenParameters}
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}

View File

@ -10,6 +10,8 @@
None
## ${docJenkinsPluginDependencies}
## Example
```groovy

View File

@ -68,6 +68,8 @@ webdriverio
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Side effects
none

View File

@ -10,6 +10,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Side effects
none

View File

@ -12,6 +12,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Example
Usage of pipeline step:

View File

@ -11,6 +11,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Side effects
Step uses `dockerExecute` inside.

View File

@ -11,6 +11,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Exceptions
none

View File

@ -79,6 +79,8 @@ testsPublishResults(
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Side effects
none

View File

@ -11,6 +11,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
The step is configured using a customer configuration file provided as
resource in an custom shared library.

View File

@ -10,6 +10,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
The step is configured using a customer configuration file provided as
resource in an custom shared library.

View File

@ -10,6 +10,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
The step is configured using a customer configuration file provided as
resource in an custom shared library.

View File

@ -8,6 +8,8 @@
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Exceptions
If you see an error like `fatal: Not a git repository (or any parent up to mount point /home/jenkins)` it is likely that your test description cannot be found.<br />

View File

@ -12,6 +12,8 @@ access protection imposed on the WhiteSource backend would simply allow access b
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}
## Exceptions
None

View File

@ -0,0 +1,71 @@
import groovy.json.JsonOutput
import groovy.json.JsonSlurper
import jenkins.model.Jenkins
unresolvableCalls = [
podTemplate:'kubernetes',
container: 'kubernetes',
docker: 'docker-plugin',
usernamePassword: 'credentials-binding',
string: 'credentials-binding',
file: 'credentials-binding',
]
node() {
stage('Resolve Plugins') {
try {
resolvePlugins()
} catch(Exception e) {
def result = System.getenv()['result']
new File(new File(result).getParentFile(), 'FAILURE').text = "${e.getMessage()}"
throw e
}
}
}
def resolvePlugins() {
def stepCallMapping = new JsonSlurper().parseText(new File(System.getenv()['calls']).text)
def stepPluginMapping = [:]
println "[INFO] Resolving plugins ..."
for(def step in stepCallMapping) {
def resolvedPlugins = [:]
for(def call in step.value) {
def resolvedPlugin = resolvePlugin(call)
if (! resolvedPlugin) {
resolvedPlugin = unresolvableCalls[call]
if(! resolvedPlugin) resolvedPlugin = 'UNIDENTIFIED'
}
if(resolvedPlugins[resolvedPlugin] == null)
resolvedPlugins[resolvedPlugin] = (Set)[]
resolvedPlugins[resolvedPlugin] << call
stepPluginMapping.put(step.key,resolvedPlugins)
}
}
def result = System.getenv()['result']
new File(result).write(new JsonOutput().toJson(stepPluginMapping))
println "[INFO] plugins resolved. Result: ${result}."
}
def resolvePlugin(call) {
def plugins = Jenkins.get().pluginManager.getPlugins()
def s = new org.jenkinsci.plugins.workflow.cps.Snippetizer()
def pDescs = s.getQuasiDescriptors(false)
for(def pd in pDescs) {
if(pd.getSymbol() == call)
return pd.real.plugin?.shortName
}
return null
}