1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-02-05 13:25:19 +02:00

add step for Synopsis Detect Scans (#690)

* add step for Synopsis Detect Scans

tool was formerly calles Blackduck Hub Detect.
Details about the tool can be found here: https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/62423113/Synopsys+Detect
This commit is contained in:
Oliver Nocon 2019-05-08 11:36:01 +02:00 committed by GitHub
parent b66d95fb1c
commit a67f850fdb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 455 additions and 117 deletions

View File

@ -88,7 +88,7 @@ class Helper {
static getConfigHelper(classLoader, roots, script) {
def compilerConfig = new CompilerConfiguration()
compilerConfig.setClasspathList( roots )
compilerConfig.setClasspathList( roots )
new GroovyClassLoader(classLoader, compilerConfig, true)
.parseClass(new File(projectRoot, 'src/com/sap/piper/ConfigurationHelper.groovy'))
@ -210,114 +210,114 @@ class Helper {
f.eachLine {
line ->
if(line ==~ /.*dependingOn.*/) {
def dependentConfigKey = (line =~ /.*dependingOn\('(.*)'\).mixin\('(.*)'/)[0][1]
def configKey = (line =~ /.*dependingOn\('(.*)'\).mixin\('(.*)'/)[0][2]
if(! step.dependentConfig[configKey]) {
step.dependentConfig[configKey] = []
if(line ==~ /.*dependingOn.*/) {
def dependentConfigKey = (line =~ /.*dependingOn\('(.*)'\).mixin\('(.*)'/)[0][1]
def configKey = (line =~ /.*dependingOn\('(.*)'\).mixin\('(.*)'/)[0][2]
if(! step.dependentConfig[configKey]) {
step.dependentConfig[configKey] = []
}
step.dependentConfig[configKey] << dependentConfigKey
}
step.dependentConfig[configKey] << dependentConfigKey
}
if(docuEnd) {
docuEnd = false
if(docuEnd) {
docuEnd = false
if(isHeader(line)) {
def _docu = []
docuLines.each { _docu << it }
_docu = Helper.trim(_docu)
step.description = _docu.join('\n')
} else {
if(isHeader(line)) {
def _docu = []
docuLines.each { _docu << it }
_docu = Helper.trim(_docu)
step.description = _docu.join('\n')
} else {
def param = retrieveParameterName(line)
def param = retrieveParameterName(line)
if(!param) {
throw new RuntimeException('Cannot retrieve parameter for a comment')
if(!param) {
throw new RuntimeException('Cannot retrieve parameter for a comment')
}
def _docu = [], _value = [], _mandatory = [], _parentObject = []
docuLines.each { _docu << it }
valueLines.each { _value << it }
mandatoryLines.each { _mandatory << it }
parentObjectLines.each { _parentObject << it }
_parentObject << param
param = _parentObject*.trim().join('/').trim()
if(step.parameters[param].docu || step.parameters[param].value)
System.err << "[WARNING] There is already some documentation for parameter '${param}. Is this parameter documented twice?'\n"
step.parameters[param].docu = _docu*.trim().join(' ').trim()
step.parameters[param].value = _value*.trim().join(' ').trim()
step.parameters[param].mandatory = _mandatory*.trim().join(' ').trim()
}
docuLines.clear()
valueLines.clear()
mandatoryLines.clear()
parentObjectLines.clear()
}
if( line.trim() ==~ /^\/\*\*.*/ ) {
docu = true
}
if(docu) {
def _line = line
_line = _line.replaceAll('^\\s*', '') // leading white spaces
if(_line.startsWith('/**')) _line = _line.replaceAll('^\\/\\*\\*', '') // start comment
if(_line.startsWith('*/') || _line.trim().endsWith('*/')) _line = _line.replaceAll('^\\*/', '').replaceAll('\\*/\\s*$', '') // end comment
if(_line.startsWith('*')) _line = _line.replaceAll('^\\*', '') // continue comment
if(_line.startsWith(' ')) _line = _line.replaceAll('^\\s', '')
if(_line ==~ /.*@possibleValues.*/) {
mandatory = false // should be something like reset attributes
value = true
parentObject = false
}
// some remark for mandatory e.g. some parameters are only mandatory under certain conditions
if(_line ==~ /.*@mandatory.*/) {
value = false // should be something like reset attributes ...
mandatory = true
parentObject = false
}
// grouping config properties within a parent object for easier readability
if(_line ==~ /.*@parentConfigKey.*/) {
value = false // should be something like reset attributes ...
mandatory = false
parentObject = true
}
def _docu = [], _value = [], _mandatory = [], _parentObject = []
docuLines.each { _docu << it }
valueLines.each { _value << it }
mandatoryLines.each { _mandatory << it }
parentObjectLines.each { _parentObject << it }
_parentObject << param
param = _parentObject*.trim().join('/').trim()
if(value) {
if(_line) {
_line = (_line =~ /.*@possibleValues\s*?(.*)/)[0][1]
valueLines << _line
}
}
if(step.parameters[param].docu || step.parameters[param].value)
System.err << "[WARNING] There is already some documentation for parameter '${param}. Is this parameter documented twice?'\n"
if(mandatory) {
if(_line) {
_line = (_line =~ /.*@mandatory\s*?(.*)/)[0][1]
mandatoryLines << _line
}
}
step.parameters[param].docu = _docu*.trim().join(' ').trim()
step.parameters[param].value = _value*.trim().join(' ').trim()
step.parameters[param].mandatory = _mandatory*.trim().join(' ').trim()
if(parentObject) {
if(_line) {
_line = (_line =~ /.*@parentConfigKey\s*?(.*)/)[0][1]
parentObjectLines << _line
}
}
if(!value && !mandatory && !parentObject) {
docuLines << _line
}
}
docuLines.clear()
valueLines.clear()
mandatoryLines.clear()
parentObjectLines.clear()
}
if( line.trim() ==~ /^\/\*\*.*/ ) {
docu = true
}
if(docu) {
def _line = line
_line = _line.replaceAll('^\\s*', '') // leading white spaces
if(_line.startsWith('/**')) _line = _line.replaceAll('^\\/\\*\\*', '') // start comment
if(_line.startsWith('*/') || _line.trim().endsWith('*/')) _line = _line.replaceAll('^\\*/', '').replaceAll('\\*/\\s*$', '') // end comment
if(_line.startsWith('*')) _line = _line.replaceAll('^\\*', '') // continue comment
if(_line.startsWith(' ')) _line = _line.replaceAll('^\\s', '')
if(_line ==~ /.*@possibleValues.*/) {
mandatory = false // should be something like reset attributes
value = true
parentObject = false
}
// some remark for mandatory e.g. some parameters are only mandatory under certain conditions
if(_line ==~ /.*@mandatory.*/) {
value = false // should be something like reset attributes ...
mandatory = true
parentObject = false
}
// grouping config properties within a parent object for easier readability
if(_line ==~ /.*@parentConfigKey.*/) {
value = false // should be something like reset attributes ...
if(docu && line.trim() ==~ /^.*\*\//) {
docu = false
value = false
mandatory = false
parentObject = true
parentObject = false
docuEnd = true
}
if(value) {
if(_line) {
_line = (_line =~ /.*@possibleValues\s*?(.*)/)[0][1]
valueLines << _line
}
}
if(mandatory) {
if(_line) {
_line = (_line =~ /.*@mandatory\s*?(.*)/)[0][1]
mandatoryLines << _line
}
}
if(parentObject) {
if(_line) {
_line = (_line =~ /.*@parentConfigKey\s*?(.*)/)[0][1]
parentObjectLines << _line
}
}
if(!value && !mandatory && !parentObject) {
docuLines << _line
}
}
if(docu && line.trim() ==~ /^.*\*\//) {
docu = false
value = false
mandatory = false
parentObject = false
docuEnd = true
}
}
}
@ -405,7 +405,7 @@ class Helper {
roots = [
new File(Helper.projectRoot, "vars").getAbsolutePath(),
new File(Helper.projectRoot, "src").getAbsolutePath()
]
]
stepsDir = null
stepsDocuDir = null
@ -434,7 +434,7 @@ if(args.length >= 3 && args[2].contains('.yml')) {
if(args.length >= 3)
steps = (args as List).drop(argsDrop) // the first two entries are stepsDir and docuDir
// the other parts are considered as step names
// the other parts are considered as step names
// assign parameters
@ -558,7 +558,7 @@ def fetchMandatoryFrom(def step, def parameterName, def steps) {
}
def fetchPossibleValuesFrom(def step, def parameterName, def steps) {
return steps[step]?.parameters[parameterName]?.value ?: ''
return steps[step]?.parameters[parameterName]?.value ?: ''
}
def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) {
@ -578,8 +578,8 @@ def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) {
prepareDefaultValuesStepParams.customDefaults = customDefaults
def defaultConfig = Helper.getConfigHelper(getClass().getClassLoader(),
roots,
Helper.getDummyScript(prepareDefaultValuesStep, stepName, prepareDefaultValuesStepParams)).use()
roots,
Helper.getDummyScript(prepareDefaultValuesStep, stepName, prepareDefaultValuesStepParams)).use()
def params = [] as Set
@ -605,10 +605,10 @@ def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) {
def compatibleParams = [] as Set
if(parentObjectMappings) {
params.each {
if (parentObjectMappings[it])
compatibleParams.add(parentObjectMappings[it] + '/' + it)
else
compatibleParams.add(it)
if (parentObjectMappings[it])
compatibleParams.add(parentObjectMappings[it] + '/' + it)
else
compatibleParams.add(it)
}
if (compatibleParams)
params = compatibleParams
@ -623,16 +623,16 @@ def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) {
// ... would be better if there is no special handling required ...
step.parameters['script'] = [
docu: 'The common script environment of the Jenkinsfile running. ' +
'Typically the reference to the script calling the pipeline ' +
'step is provided with the this parameter, as in `script: this`. ' +
'This allows the function to access the ' +
'commonPipelineEnvironment for retrieving, for example, configuration parameters.',
required: true,
docu: 'The common script environment of the Jenkinsfile running. ' +
'Typically the reference to the script calling the pipeline ' +
'step is provided with the this parameter, as in `script: this`. ' +
'This allows the function to access the ' +
'commonPipelineEnvironment for retrieving, for example, configuration parameters.',
required: true,
GENERAL_CONFIG: false,
STEP_CONFIG: false
]
GENERAL_CONFIG: false,
STEP_CONFIG: false
]
// END special handling for 'script' parameter
@ -643,9 +643,9 @@ def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) {
def defaultValue = Helper.getValue(defaultConfig, it.split('/'))
def parameterProperties = [
defaultValue: defaultValue,
required: requiredParameters.contains((it as String)) && defaultValue == null
]
defaultValue: defaultValue,
required: requiredParameters.contains((it as String)) && defaultValue == null
]
step.parameters.put(it, parameterProperties)

View File

@ -0,0 +1,20 @@
# ${docGenStepName}
## ${docGenDescription}
## Prerequsites
You need to store the API token for the Detect service as _'Secret text'_ credential in your Jenkins system.
!!! note "minimum plugin requirement"
This step requires [synopsys-detect-plugin](https://github.com/jenkinsci/synopsys-detect-plugin) with at least version `2.0.0`.
## Example
```groovy
detectExecuteScan script: this, scanProperties: ['--logging.level.com.synopsys.integration=TRACE']
```
## ${docGenParameters}
## ${docGenConfiguration}

View File

@ -177,6 +177,27 @@ steps:
stashContent:
- 'tests'
testReportFilePath: 'cst-report.json'
detectExecuteScan:
detect:
projectVersion: '1'
scanners:
- signature
scanPaths:
- '.'
scanProperties:
- '--blackduck.signature.scanner.memory=4096'
- '--blackduck.timeout=6000'
- '--blackduck.trust.cert=true'
- '--detect.policy.check.fail.on.severities=BLOCKER,CRITICAL,MAJOR'
- '--detect.report.timeout=4800'
- '--logging.level.com.synopsys.integration=DEBUG'
stashContent:
- 'buildDescriptor'
- 'checkmarx'
# buildTool specific settings
golang:
dockerImage: 'golang:1.12-stretch'
dockerWorkspace: ''
dockerExecute:
dockerPullImage: true
sidecarPullImage: true

View File

@ -0,0 +1,143 @@
#!groovy
import org.junit.Before
import org.junit.Rule
import org.junit.Test
import org.junit.rules.RuleChain
import util.BasePiperTest
import util.JenkinsCredentialsRule
import util.JenkinsDockerExecuteRule
import util.JenkinsReadYamlRule
import util.JenkinsShellCallRule
import util.JenkinsStepRule
import util.Rules
import static org.hamcrest.CoreMatchers.containsString
import static org.hamcrest.CoreMatchers.is
import static org.hamcrest.Matchers.allOf
import static org.hamcrest.Matchers.hasItem
import static org.hamcrest.Matchers.not
import static org.junit.Assert.assertThat
class DetectExecuteScanTest extends BasePiperTest {
private JenkinsDockerExecuteRule dockerRule = new JenkinsDockerExecuteRule(this)
private JenkinsShellCallRule shellRule = new JenkinsShellCallRule(this)
private JenkinsStepRule stepRule = new JenkinsStepRule(this)
private String detectProperties = ''
@Rule
public RuleChain rules = Rules
.getCommonRules(this)
.around(new JenkinsReadYamlRule(this))
.around(shellRule)
.around(dockerRule)
.around(stepRule)
.around(new JenkinsCredentialsRule(this)
.withCredentials('testCredentials', 'testToken')
)
@Before
void init() {
detectProperties = ''
helper.registerAllowedMethod('synopsys_detect', [String.class], {s ->
detectProperties = s
})
}
@Test
void testDetectDefault() {
stepRule.step.detectExecuteScan([
apiTokenCredentialsId: 'testCredentials',
projectName: 'testProject',
serverUrl: 'https://test.blackducksoftware.com',
juStabUtils: utils,
script: nullScript
])
//ToDo: assert unstashing
assertThat(detectProperties, containsString("--detect.project.name='testProject'"))
assertThat(detectProperties, containsString("--detect.project.version.name='1'"))
assertThat(detectProperties, containsString("--blackduck.url=https://test.blackducksoftware.com"))
assertThat(detectProperties, containsString("--blackduck.api.token=testToken"))
assertThat(detectProperties, containsString("--detect.blackduck.signature.scanner.paths=."))
assertThat(detectProperties, containsString("--blackduck.signature.scanner.memory=4096"))
assertThat(detectProperties, containsString("--blackduck.timeout=6000"))
assertThat(detectProperties, containsString("--blackduck.trust.cert=true"))
assertThat(detectProperties, containsString("--detect.report.timeout=4800"))
}
@Test
void testDetectCustomPaths() {
stepRule.step.detectExecuteScan([
apiTokenCredentialsId: 'testCredentials',
projectName: 'testProject',
scanPaths: ['test1/', 'test2/'],
serverUrl: 'https://test.blackducksoftware.com',
juStabUtils: utils,
script: nullScript
])
assertThat(detectProperties, containsString("--detect.blackduck.signature.scanner.paths=test1/,test2/"))
}
@Test
void testDetectSourceScanOnly() {
stepRule.step.detectExecuteScan([
apiTokenCredentialsId: 'testCredentials',
projectName: 'testProject',
scanners: ['source'],
serverUrl: 'https://test.blackducksoftware.com',
juStabUtils: utils,
script: nullScript
])
assertThat(detectProperties, not(containsString("--detect.blackduck.signature.scanner.paths=.")))
assertThat(detectProperties, containsString("--detect.source.path=."))
}
@Test
void testDetectGolang() {
stepRule.step.detectExecuteScan([
buildTool: 'golang',
apiTokenCredentialsId: 'testCredentials',
projectName: 'testProject',
serverUrl: 'https://test.blackducksoftware.com',
juStabUtils: utils,
script: nullScript
])
assertThat(dockerRule.dockerParams.dockerImage, is('golang:1.12-stretch'))
assertThat(dockerRule.dockerParams.dockerWorkspace, is(''))
assertThat(dockerRule.dockerParams.stashContent, allOf(hasItem('buildDescriptor'),hasItem('checkmarx')))
assertThat(shellRule.shell, hasItem('curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh'))
assertThat(shellRule.shell, hasItem('ln --symbolic $(pwd) $GOPATH/src/hub'))
assertThat(shellRule.shell, hasItem('cd $GOPATH/src/hub && dep ensure'))
}
@Test
void testCustomScanProperties() {
def detectProps = [
'--blackduck.signature.scanner.memory=1024'
]
stepRule.step.detectExecuteScan([
//scanProperties: detectProps,
scanProperties: ['--blackduck.signature.scanner.memory=1024', '--myNewOne'],
apiTokenCredentialsId: 'testCredentials',
projectName: 'testProject',
serverUrl: 'https://test.blackducksoftware.com',
juStabUtils: utils,
script: nullScript
])
assertThat(detectProperties, containsString("--detect.project.name='testProject'"))
assertThat(detectProperties, containsString("--detect.project.version.name='1'"))
assertThat(detectProperties, containsString("--blackduck.signature.scanner.memory=1024"))
assertThat(detectProperties, not(containsString("--blackduck.signature.scanner.memory=4096")))
assertThat(detectProperties, not(containsString("--detect.report.timeout=4800")))
assertThat(detectProperties, containsString("--myNewOne"))
}
}

View File

@ -0,0 +1,154 @@
import com.sap.piper.GenerateDocumentation
import com.sap.piper.Utils
import com.sap.piper.ConfigurationHelper
import groovy.transform.Field
import static com.sap.piper.Prerequisites.checkScript
@Field String STEP_NAME = getClass().getName()
@Field Set GENERAL_CONFIG_KEYS = [
'detect',
/**
* Jenkins 'Secret text' credentials ID containing the API token used to authenticate with the Synopsis Detect (formerly BlackDuck) Server.
* @parentConfigKey detect
*/
'apiTokenCredentialsId',
/**
* Defines the tool which is used for building the artifact.<br />
* Currently, it is possible to select two behaviors of the step:
* <br />
* 1. Golang-specific behavior (`buildTool: golang`). Assumption here is that project uses the dependency management tool _dep_<br />
* 2. Custom-specific behavior for all other values of `buildTool`
*
* @possibleValues `golang`, any other build tool
*/
'buildTool',
/**
* Name of the Synopsis Detect (formerly BlackDuck) project.
* @parentConfigKey detect
*/
'projectName',
/**
* Version of the Synopsis Detect (formerly BlackDuck) project.
* @parentConfigKey detect
*/
'projectVersion',
/**
* List of paths which should be scanned by the Synopsis Detect (formerly BlackDuck) scan.
* @parentConfigKey detect
*/
'scanPaths',
/**
* Properties passed to the Synopsis Detect (formerly BlackDuck) scan. You can find details in the [Synopsis Detect documentation](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/622846/Using+Synopsys+Detect+Properties)
* @parentConfigKey detect
*/
'scanProperties',
/**
* List of scanners to be used for Synopsis Detect (formerly BlackDuck) scan.
* @possibleValues `['signature']`
* @parentConfigKey detect
*/
'scanners',
/**
* Server url to the Synopsis Detect (formerly BlackDuck) Server.
* @parentConfigKey detect
*/
'serverUrl'
]
@Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS.plus([
/** @see dockerExecute */
'dockerImage',
/** @see dockerExecute */
'dockerWorkspace',
/** If specific stashes should be considered for the scan, their names need to be passed via the parameter `stashContent`. */
'stashContent'
])
@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS
@Field Map CONFIG_KEY_COMPATIBILITY = [
detect: [
apiTokenCredentialsId: 'apiTokenCredentialsId',
projectName: 'projectName',
projectVersion: 'projectVersion',
scanners: 'scanners',
scanPaths: 'scanPaths',
scanProperties: 'scanProperties',
serverUrl: 'serverUrl'
]
]
/**
* This step executes [Synopsis Detect](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/62423113/Synopsys+Detect) scans.
*/
@GenerateDocumentation
void call(Map parameters = [:]) {
handlePipelineStepErrors (stepName: STEP_NAME, stepParameters: parameters) {
def script = checkScript(this, parameters) ?: this
def utils = parameters.juStabUtils ?: new Utils()
// load default & individual configuration
Map config = ConfigurationHelper.newInstance(this)
.loadStepDefaults()
.mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY)
.mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS,CONFIG_KEY_COMPATIBILITY)
.mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName?:env.STAGE_NAME, STEP_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY)
.mixin(parameters, PARAMETER_KEYS, CONFIG_KEY_COMPATIBILITY)
.dependingOn('buildTool').mixin('dockerImage')
.dependingOn('buildTool').mixin('dockerWorkspace')
.withMandatoryProperty('detect/apiTokenCredentialsId')
.withMandatoryProperty('detect/projectName')
.withMandatoryProperty('detect/projectVersion')
.use()
config.stashContent = utils.unstashAll(config.stashContent)
script.commonPipelineEnvironment.setInfluxStepData('detect', false)
utils.pushToSWA([
step: STEP_NAME,
stepParamKey1: 'buildTool',
stepParam1: config.buildTool ?: 'default'
], config)
//prepare Hub Detect execution using package manager
switch (config.buildTool) {
case 'golang':
dockerExecute(script: script, dockerImage: config.dockerImage, dockerWorkspace: config.dockerWorkspace, stashContent: config.stashContent) {
sh 'curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh'
sh 'ln --symbolic $(pwd) $GOPATH/src/hub'
sh 'cd $GOPATH/src/hub && dep ensure'
}
break
default:
//no additional tasks are performed
echo "[${STEP_NAME}] No preparation steps performed for scan. Please make sure to properly set configuration for `detect.scanProperties`"
}
withCredentials ([string(
credentialsId: config.detect.apiTokenCredentialsId,
variable: 'detectApiToken'
)]) {
def authentication = "--blackduck.api.token=${detectApiToken}"
config.detect.scanProperties += [
"--detect.project.name='${config.detect.projectName}'",
"--detect.project.version.name='${config.detect.projectVersion}'",
"--detect.code.location.name='${config.detect.projectName}/${config.detect.projectVersion}'",
"--blackduck.url=${config.detect.serverUrl}",
]
if ('signature' in config.detect.scanners) [
config.detect.scanProperties.add("--detect.blackduck.signature.scanner.paths=${config.detect.scanPaths.join(',')}")
]
if ('source' in config.detect.scanners) [
config.detect.scanProperties.add("--detect.source.path=${config.detect.scanPaths[0]}")
]
def detectProperties = config.detect.scanProperties.join(' ') + " ${authentication}"
echo "[${STEP_NAME}] Running with following Detect configuration: ${detectProperties}"
synopsys_detect detectProperties
script.commonPipelineEnvironment.setInfluxStepData('detect', true)
}
}
}