diff --git a/documentation/bin/createDocu.groovy b/documentation/bin/createDocu.groovy index bccb56cfa..83bce877d 100644 --- a/documentation/bin/createDocu.groovy +++ b/documentation/bin/createDocu.groovy @@ -88,7 +88,7 @@ class Helper { static getConfigHelper(classLoader, roots, script) { def compilerConfig = new CompilerConfiguration() - compilerConfig.setClasspathList( roots ) + compilerConfig.setClasspathList( roots ) new GroovyClassLoader(classLoader, compilerConfig, true) .parseClass(new File(projectRoot, 'src/com/sap/piper/ConfigurationHelper.groovy')) @@ -210,114 +210,114 @@ class Helper { f.eachLine { line -> - if(line ==~ /.*dependingOn.*/) { - def dependentConfigKey = (line =~ /.*dependingOn\('(.*)'\).mixin\('(.*)'/)[0][1] - def configKey = (line =~ /.*dependingOn\('(.*)'\).mixin\('(.*)'/)[0][2] - if(! step.dependentConfig[configKey]) { - step.dependentConfig[configKey] = [] + if(line ==~ /.*dependingOn.*/) { + def dependentConfigKey = (line =~ /.*dependingOn\('(.*)'\).mixin\('(.*)'/)[0][1] + def configKey = (line =~ /.*dependingOn\('(.*)'\).mixin\('(.*)'/)[0][2] + if(! step.dependentConfig[configKey]) { + step.dependentConfig[configKey] = [] + } + step.dependentConfig[configKey] << dependentConfigKey } - step.dependentConfig[configKey] << dependentConfigKey - } - if(docuEnd) { - docuEnd = false + if(docuEnd) { + docuEnd = false - if(isHeader(line)) { - def _docu = [] - docuLines.each { _docu << it } - _docu = Helper.trim(_docu) - step.description = _docu.join('\n') - } else { + if(isHeader(line)) { + def _docu = [] + docuLines.each { _docu << it } + _docu = Helper.trim(_docu) + step.description = _docu.join('\n') + } else { - def param = retrieveParameterName(line) + def param = retrieveParameterName(line) - if(!param) { - throw new RuntimeException('Cannot retrieve parameter for a comment') + if(!param) { + throw new RuntimeException('Cannot retrieve parameter for a comment') + } + + def _docu = [], _value = [], _mandatory = [], _parentObject = [] + docuLines.each { _docu << it } + valueLines.each { _value << it } + mandatoryLines.each { _mandatory << it } + parentObjectLines.each { _parentObject << it } + _parentObject << param + param = _parentObject*.trim().join('/').trim() + + if(step.parameters[param].docu || step.parameters[param].value) + System.err << "[WARNING] There is already some documentation for parameter '${param}. Is this parameter documented twice?'\n" + + step.parameters[param].docu = _docu*.trim().join(' ').trim() + step.parameters[param].value = _value*.trim().join(' ').trim() + step.parameters[param].mandatory = _mandatory*.trim().join(' ').trim() + } + docuLines.clear() + valueLines.clear() + mandatoryLines.clear() + parentObjectLines.clear() + } + + if( line.trim() ==~ /^\/\*\*.*/ ) { + docu = true + } + + if(docu) { + def _line = line + _line = _line.replaceAll('^\\s*', '') // leading white spaces + if(_line.startsWith('/**')) _line = _line.replaceAll('^\\/\\*\\*', '') // start comment + if(_line.startsWith('*/') || _line.trim().endsWith('*/')) _line = _line.replaceAll('^\\*/', '').replaceAll('\\*/\\s*$', '') // end comment + if(_line.startsWith('*')) _line = _line.replaceAll('^\\*', '') // continue comment + if(_line.startsWith(' ')) _line = _line.replaceAll('^\\s', '') + if(_line ==~ /.*@possibleValues.*/) { + mandatory = false // should be something like reset attributes + value = true + parentObject = false + } + // some remark for mandatory e.g. some parameters are only mandatory under certain conditions + if(_line ==~ /.*@mandatory.*/) { + value = false // should be something like reset attributes ... + mandatory = true + parentObject = false + } + // grouping config properties within a parent object for easier readability + if(_line ==~ /.*@parentConfigKey.*/) { + value = false // should be something like reset attributes ... + mandatory = false + parentObject = true } - def _docu = [], _value = [], _mandatory = [], _parentObject = [] - docuLines.each { _docu << it } - valueLines.each { _value << it } - mandatoryLines.each { _mandatory << it } - parentObjectLines.each { _parentObject << it } - _parentObject << param - param = _parentObject*.trim().join('/').trim() + if(value) { + if(_line) { + _line = (_line =~ /.*@possibleValues\s*?(.*)/)[0][1] + valueLines << _line + } + } - if(step.parameters[param].docu || step.parameters[param].value) - System.err << "[WARNING] There is already some documentation for parameter '${param}. Is this parameter documented twice?'\n" + if(mandatory) { + if(_line) { + _line = (_line =~ /.*@mandatory\s*?(.*)/)[0][1] + mandatoryLines << _line + } + } - step.parameters[param].docu = _docu*.trim().join(' ').trim() - step.parameters[param].value = _value*.trim().join(' ').trim() - step.parameters[param].mandatory = _mandatory*.trim().join(' ').trim() + if(parentObject) { + if(_line) { + _line = (_line =~ /.*@parentConfigKey\s*?(.*)/)[0][1] + parentObjectLines << _line + } + } + + if(!value && !mandatory && !parentObject) { + docuLines << _line + } } - docuLines.clear() - valueLines.clear() - mandatoryLines.clear() - parentObjectLines.clear() - } - if( line.trim() ==~ /^\/\*\*.*/ ) { - docu = true - } - - if(docu) { - def _line = line - _line = _line.replaceAll('^\\s*', '') // leading white spaces - if(_line.startsWith('/**')) _line = _line.replaceAll('^\\/\\*\\*', '') // start comment - if(_line.startsWith('*/') || _line.trim().endsWith('*/')) _line = _line.replaceAll('^\\*/', '').replaceAll('\\*/\\s*$', '') // end comment - if(_line.startsWith('*')) _line = _line.replaceAll('^\\*', '') // continue comment - if(_line.startsWith(' ')) _line = _line.replaceAll('^\\s', '') - if(_line ==~ /.*@possibleValues.*/) { - mandatory = false // should be something like reset attributes - value = true - parentObject = false - } - // some remark for mandatory e.g. some parameters are only mandatory under certain conditions - if(_line ==~ /.*@mandatory.*/) { - value = false // should be something like reset attributes ... - mandatory = true - parentObject = false - } - // grouping config properties within a parent object for easier readability - if(_line ==~ /.*@parentConfigKey.*/) { - value = false // should be something like reset attributes ... + if(docu && line.trim() ==~ /^.*\*\//) { + docu = false + value = false mandatory = false - parentObject = true + parentObject = false + docuEnd = true } - - if(value) { - if(_line) { - _line = (_line =~ /.*@possibleValues\s*?(.*)/)[0][1] - valueLines << _line - } - } - - if(mandatory) { - if(_line) { - _line = (_line =~ /.*@mandatory\s*?(.*)/)[0][1] - mandatoryLines << _line - } - } - - if(parentObject) { - if(_line) { - _line = (_line =~ /.*@parentConfigKey\s*?(.*)/)[0][1] - parentObjectLines << _line - } - } - - if(!value && !mandatory && !parentObject) { - docuLines << _line - } - } - - if(docu && line.trim() ==~ /^.*\*\//) { - docu = false - value = false - mandatory = false - parentObject = false - docuEnd = true - } } } @@ -405,7 +405,7 @@ class Helper { roots = [ new File(Helper.projectRoot, "vars").getAbsolutePath(), new File(Helper.projectRoot, "src").getAbsolutePath() - ] +] stepsDir = null stepsDocuDir = null @@ -434,7 +434,7 @@ if(args.length >= 3 && args[2].contains('.yml')) { if(args.length >= 3) steps = (args as List).drop(argsDrop) // the first two entries are stepsDir and docuDir - // the other parts are considered as step names +// the other parts are considered as step names // assign parameters @@ -558,7 +558,7 @@ def fetchMandatoryFrom(def step, def parameterName, def steps) { } def fetchPossibleValuesFrom(def step, def parameterName, def steps) { - return steps[step]?.parameters[parameterName]?.value ?: '' + return steps[step]?.parameters[parameterName]?.value ?: '' } def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) { @@ -578,8 +578,8 @@ def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) { prepareDefaultValuesStepParams.customDefaults = customDefaults def defaultConfig = Helper.getConfigHelper(getClass().getClassLoader(), - roots, - Helper.getDummyScript(prepareDefaultValuesStep, stepName, prepareDefaultValuesStepParams)).use() + roots, + Helper.getDummyScript(prepareDefaultValuesStep, stepName, prepareDefaultValuesStepParams)).use() def params = [] as Set @@ -605,10 +605,10 @@ def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) { def compatibleParams = [] as Set if(parentObjectMappings) { params.each { - if (parentObjectMappings[it]) - compatibleParams.add(parentObjectMappings[it] + '/' + it) - else - compatibleParams.add(it) + if (parentObjectMappings[it]) + compatibleParams.add(parentObjectMappings[it] + '/' + it) + else + compatibleParams.add(it) } if (compatibleParams) params = compatibleParams @@ -623,16 +623,16 @@ def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) { // ... would be better if there is no special handling required ... step.parameters['script'] = [ - docu: 'The common script environment of the Jenkinsfile running. ' + - 'Typically the reference to the script calling the pipeline ' + - 'step is provided with the this parameter, as in `script: this`. ' + - 'This allows the function to access the ' + - 'commonPipelineEnvironment for retrieving, for example, configuration parameters.', - required: true, + docu: 'The common script environment of the Jenkinsfile running. ' + + 'Typically the reference to the script calling the pipeline ' + + 'step is provided with the this parameter, as in `script: this`. ' + + 'This allows the function to access the ' + + 'commonPipelineEnvironment for retrieving, for example, configuration parameters.', + required: true, - GENERAL_CONFIG: false, - STEP_CONFIG: false - ] + GENERAL_CONFIG: false, + STEP_CONFIG: false + ] // END special handling for 'script' parameter @@ -643,9 +643,9 @@ def handleStep(stepName, prepareDefaultValuesStep, gse, customDefaults) { def defaultValue = Helper.getValue(defaultConfig, it.split('/')) def parameterProperties = [ - defaultValue: defaultValue, - required: requiredParameters.contains((it as String)) && defaultValue == null - ] + defaultValue: defaultValue, + required: requiredParameters.contains((it as String)) && defaultValue == null + ] step.parameters.put(it, parameterProperties) diff --git a/documentation/docs/steps/detectExecuteScan.md b/documentation/docs/steps/detectExecuteScan.md new file mode 100644 index 000000000..e92b1db5b --- /dev/null +++ b/documentation/docs/steps/detectExecuteScan.md @@ -0,0 +1,20 @@ +# ${docGenStepName} + +## ${docGenDescription} + +## Prerequsites + +You need to store the API token for the Detect service as _'Secret text'_ credential in your Jenkins system. + +!!! note "minimum plugin requirement" + This step requires [synopsys-detect-plugin](https://github.com/jenkinsci/synopsys-detect-plugin) with at least version `2.0.0`. + +## Example + +```groovy +detectExecuteScan script: this, scanProperties: ['--logging.level.com.synopsys.integration=TRACE'] +``` + +## ${docGenParameters} + +## ${docGenConfiguration} diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 26af56c35..1bb4c467c 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -177,6 +177,27 @@ steps: stashContent: - 'tests' testReportFilePath: 'cst-report.json' + detectExecuteScan: + detect: + projectVersion: '1' + scanners: + - signature + scanPaths: + - '.' + scanProperties: + - '--blackduck.signature.scanner.memory=4096' + - '--blackduck.timeout=6000' + - '--blackduck.trust.cert=true' + - '--detect.policy.check.fail.on.severities=BLOCKER,CRITICAL,MAJOR' + - '--detect.report.timeout=4800' + - '--logging.level.com.synopsys.integration=DEBUG' + stashContent: + - 'buildDescriptor' + - 'checkmarx' + # buildTool specific settings + golang: + dockerImage: 'golang:1.12-stretch' + dockerWorkspace: '' dockerExecute: dockerPullImage: true sidecarPullImage: true diff --git a/test/groovy/DetectExecuteScanTest.groovy b/test/groovy/DetectExecuteScanTest.groovy new file mode 100644 index 000000000..800e68d35 --- /dev/null +++ b/test/groovy/DetectExecuteScanTest.groovy @@ -0,0 +1,143 @@ +#!groovy +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.RuleChain +import util.BasePiperTest +import util.JenkinsCredentialsRule +import util.JenkinsDockerExecuteRule +import util.JenkinsReadYamlRule +import util.JenkinsShellCallRule +import util.JenkinsStepRule +import util.Rules + +import static org.hamcrest.CoreMatchers.containsString +import static org.hamcrest.CoreMatchers.is +import static org.hamcrest.Matchers.allOf +import static org.hamcrest.Matchers.hasItem +import static org.hamcrest.Matchers.not +import static org.junit.Assert.assertThat + +class DetectExecuteScanTest extends BasePiperTest { + + private JenkinsDockerExecuteRule dockerRule = new JenkinsDockerExecuteRule(this) + private JenkinsShellCallRule shellRule = new JenkinsShellCallRule(this) + private JenkinsStepRule stepRule = new JenkinsStepRule(this) + + private String detectProperties = '' + + @Rule + public RuleChain rules = Rules + .getCommonRules(this) + .around(new JenkinsReadYamlRule(this)) + .around(shellRule) + .around(dockerRule) + .around(stepRule) + .around(new JenkinsCredentialsRule(this) + .withCredentials('testCredentials', 'testToken') + ) + + @Before + void init() { + + detectProperties = '' + helper.registerAllowedMethod('synopsys_detect', [String.class], {s -> + detectProperties = s + }) + } + + @Test + void testDetectDefault() { + stepRule.step.detectExecuteScan([ + apiTokenCredentialsId: 'testCredentials', + projectName: 'testProject', + serverUrl: 'https://test.blackducksoftware.com', + juStabUtils: utils, + script: nullScript + ]) + + //ToDo: assert unstashing + + assertThat(detectProperties, containsString("--detect.project.name='testProject'")) + assertThat(detectProperties, containsString("--detect.project.version.name='1'")) + assertThat(detectProperties, containsString("--blackduck.url=https://test.blackducksoftware.com")) + assertThat(detectProperties, containsString("--blackduck.api.token=testToken")) + assertThat(detectProperties, containsString("--detect.blackduck.signature.scanner.paths=.")) + assertThat(detectProperties, containsString("--blackduck.signature.scanner.memory=4096")) + assertThat(detectProperties, containsString("--blackduck.timeout=6000")) + assertThat(detectProperties, containsString("--blackduck.trust.cert=true")) + assertThat(detectProperties, containsString("--detect.report.timeout=4800")) + } + + @Test + void testDetectCustomPaths() { + stepRule.step.detectExecuteScan([ + apiTokenCredentialsId: 'testCredentials', + projectName: 'testProject', + scanPaths: ['test1/', 'test2/'], + serverUrl: 'https://test.blackducksoftware.com', + juStabUtils: utils, + script: nullScript + ]) + + assertThat(detectProperties, containsString("--detect.blackduck.signature.scanner.paths=test1/,test2/")) + } + + @Test + void testDetectSourceScanOnly() { + stepRule.step.detectExecuteScan([ + apiTokenCredentialsId: 'testCredentials', + projectName: 'testProject', + scanners: ['source'], + serverUrl: 'https://test.blackducksoftware.com', + juStabUtils: utils, + script: nullScript + ]) + + assertThat(detectProperties, not(containsString("--detect.blackduck.signature.scanner.paths=."))) + assertThat(detectProperties, containsString("--detect.source.path=.")) + } + + @Test + void testDetectGolang() { + stepRule.step.detectExecuteScan([ + buildTool: 'golang', + apiTokenCredentialsId: 'testCredentials', + projectName: 'testProject', + serverUrl: 'https://test.blackducksoftware.com', + juStabUtils: utils, + script: nullScript + ]) + + assertThat(dockerRule.dockerParams.dockerImage, is('golang:1.12-stretch')) + assertThat(dockerRule.dockerParams.dockerWorkspace, is('')) + assertThat(dockerRule.dockerParams.stashContent, allOf(hasItem('buildDescriptor'),hasItem('checkmarx'))) + + assertThat(shellRule.shell, hasItem('curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh')) + assertThat(shellRule.shell, hasItem('ln --symbolic $(pwd) $GOPATH/src/hub')) + assertThat(shellRule.shell, hasItem('cd $GOPATH/src/hub && dep ensure')) + } + + @Test + void testCustomScanProperties() { + def detectProps = [ + '--blackduck.signature.scanner.memory=1024' + ] + stepRule.step.detectExecuteScan([ + //scanProperties: detectProps, + scanProperties: ['--blackduck.signature.scanner.memory=1024', '--myNewOne'], + apiTokenCredentialsId: 'testCredentials', + projectName: 'testProject', + serverUrl: 'https://test.blackducksoftware.com', + juStabUtils: utils, + script: nullScript + ]) + + assertThat(detectProperties, containsString("--detect.project.name='testProject'")) + assertThat(detectProperties, containsString("--detect.project.version.name='1'")) + assertThat(detectProperties, containsString("--blackduck.signature.scanner.memory=1024")) + assertThat(detectProperties, not(containsString("--blackduck.signature.scanner.memory=4096"))) + assertThat(detectProperties, not(containsString("--detect.report.timeout=4800"))) + assertThat(detectProperties, containsString("--myNewOne")) + } +} diff --git a/vars/detectExecuteScan.groovy b/vars/detectExecuteScan.groovy new file mode 100644 index 000000000..bb08c68e5 --- /dev/null +++ b/vars/detectExecuteScan.groovy @@ -0,0 +1,154 @@ +import com.sap.piper.GenerateDocumentation +import com.sap.piper.Utils +import com.sap.piper.ConfigurationHelper + +import groovy.transform.Field + +import static com.sap.piper.Prerequisites.checkScript + +@Field String STEP_NAME = getClass().getName() +@Field Set GENERAL_CONFIG_KEYS = [ + 'detect', + /** + * Jenkins 'Secret text' credentials ID containing the API token used to authenticate with the Synopsis Detect (formerly BlackDuck) Server. + * @parentConfigKey detect + */ + 'apiTokenCredentialsId', + /** + * Defines the tool which is used for building the artifact.
+ * Currently, it is possible to select two behaviors of the step: + *
+ * 1. Golang-specific behavior (`buildTool: golang`). Assumption here is that project uses the dependency management tool _dep_
+ * 2. Custom-specific behavior for all other values of `buildTool` + * + * @possibleValues `golang`, any other build tool + */ + 'buildTool', + /** + * Name of the Synopsis Detect (formerly BlackDuck) project. + * @parentConfigKey detect + */ + 'projectName', + /** + * Version of the Synopsis Detect (formerly BlackDuck) project. + * @parentConfigKey detect + */ + 'projectVersion', + /** + * List of paths which should be scanned by the Synopsis Detect (formerly BlackDuck) scan. + * @parentConfigKey detect + */ + 'scanPaths', + /** + * Properties passed to the Synopsis Detect (formerly BlackDuck) scan. You can find details in the [Synopsis Detect documentation](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/622846/Using+Synopsys+Detect+Properties) + * @parentConfigKey detect + */ + 'scanProperties', + /** + * List of scanners to be used for Synopsis Detect (formerly BlackDuck) scan. + * @possibleValues `['signature']` + * @parentConfigKey detect + */ + 'scanners', + /** + * Server url to the Synopsis Detect (formerly BlackDuck) Server. + * @parentConfigKey detect + */ + 'serverUrl' +] +@Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS.plus([ + /** @see dockerExecute */ + 'dockerImage', + /** @see dockerExecute */ + 'dockerWorkspace', + /** If specific stashes should be considered for the scan, their names need to be passed via the parameter `stashContent`. */ + 'stashContent' +]) +@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS + +@Field Map CONFIG_KEY_COMPATIBILITY = [ + detect: [ + apiTokenCredentialsId: 'apiTokenCredentialsId', + projectName: 'projectName', + projectVersion: 'projectVersion', + scanners: 'scanners', + scanPaths: 'scanPaths', + scanProperties: 'scanProperties', + serverUrl: 'serverUrl' + ] +] + +/** + * This step executes [Synopsis Detect](https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/62423113/Synopsys+Detect) scans. + */ +@GenerateDocumentation +void call(Map parameters = [:]) { + handlePipelineStepErrors (stepName: STEP_NAME, stepParameters: parameters) { + def script = checkScript(this, parameters) ?: this + def utils = parameters.juStabUtils ?: new Utils() + // load default & individual configuration + Map config = ConfigurationHelper.newInstance(this) + .loadStepDefaults() + .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY) + .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS,CONFIG_KEY_COMPATIBILITY) + .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName?:env.STAGE_NAME, STEP_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY) + .mixin(parameters, PARAMETER_KEYS, CONFIG_KEY_COMPATIBILITY) + .dependingOn('buildTool').mixin('dockerImage') + .dependingOn('buildTool').mixin('dockerWorkspace') + .withMandatoryProperty('detect/apiTokenCredentialsId') + .withMandatoryProperty('detect/projectName') + .withMandatoryProperty('detect/projectVersion') + .use() + + config.stashContent = utils.unstashAll(config.stashContent) + + script.commonPipelineEnvironment.setInfluxStepData('detect', false) + + utils.pushToSWA([ + step: STEP_NAME, + stepParamKey1: 'buildTool', + stepParam1: config.buildTool ?: 'default' + ], config) + + //prepare Hub Detect execution using package manager + switch (config.buildTool) { + case 'golang': + dockerExecute(script: script, dockerImage: config.dockerImage, dockerWorkspace: config.dockerWorkspace, stashContent: config.stashContent) { + sh 'curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh' + sh 'ln --symbolic $(pwd) $GOPATH/src/hub' + sh 'cd $GOPATH/src/hub && dep ensure' + } + break + default: + //no additional tasks are performed + echo "[${STEP_NAME}] No preparation steps performed for scan. Please make sure to properly set configuration for `detect.scanProperties`" + } + + withCredentials ([string( + credentialsId: config.detect.apiTokenCredentialsId, + variable: 'detectApiToken' + )]) { + def authentication = "--blackduck.api.token=${detectApiToken}" + config.detect.scanProperties += [ + "--detect.project.name='${config.detect.projectName}'", + "--detect.project.version.name='${config.detect.projectVersion}'", + "--detect.code.location.name='${config.detect.projectName}/${config.detect.projectVersion}'", + "--blackduck.url=${config.detect.serverUrl}", + ] + + if ('signature' in config.detect.scanners) [ + config.detect.scanProperties.add("--detect.blackduck.signature.scanner.paths=${config.detect.scanPaths.join(',')}") + ] + + if ('source' in config.detect.scanners) [ + config.detect.scanProperties.add("--detect.source.path=${config.detect.scanPaths[0]}") + ] + + def detectProperties = config.detect.scanProperties.join(' ') + " ${authentication}" + + echo "[${STEP_NAME}] Running with following Detect configuration: ${detectProperties}" + synopsys_detect detectProperties + script.commonPipelineEnvironment.setInfluxStepData('detect', true) + } + } +}