1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-03-03 15:02:35 +02:00

Cleanup and alignment with internal step

This commit is contained in:
Sven Merk 2019-03-05 13:59:27 +01:00
parent 34857f112f
commit 53491ce956
5 changed files with 175 additions and 213 deletions

View File

@ -259,6 +259,7 @@ steps:
parallelLimit: 15
licensingVulnerabilities: true
securityVulnerabilities: true
cvssSeverityLimit: -1
reporting: true
vulnerabilityReportFileName: 'piper_whitesource_vulnerability_report'
vulnerabilityReportTitle: 'WhiteSource Security Vulnerability Report'

View File

@ -6,8 +6,6 @@ import java.security.MessageDigest
class WhitesourceConfigurationHelper implements Serializable {
private static def SCALA_CONTENT_KEY = "@__content"
static def extendUAConfigurationFile(script, utils, config, path) {
def mapping = []
def parsingClosure = { fileReadPath -> return script.readProperties (file: fileReadPath) }
@ -81,62 +79,6 @@ class WhitesourceConfigurationHelper implements Serializable {
rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure)
}
static def extendConfigurationFile(script, utils, config, path) {
def mapping = [:]
def parsingClosure
def serializationClosure
def inputFile = config.configFilePath.replaceFirst('\\./', '')
def suffix = MessageDigest.getInstance("MD5").digest(config.configFilePath.bytes).encodeHex().toString()
def targetFile = "${inputFile}.${suffix}"
switch (config.scanType) {
case 'unifiedAgent':
case 'fileAgent':
mapping = [
[name: 'apiKey', value: config.orgToken, warnIfPresent: true],
[name: 'productName', value: config.productName],
[name: 'productToken', value: config.productToken, omitIfPresent: 'projectToken'],
[name: 'userKey', value: config.userKey, warnIfPresent: true]
]
parsingClosure = { fileReadPath -> return script.readProperties (file: fileReadPath) }
serializationClosure = { configuration -> serializeUAConfig(configuration) }
break
case 'npm':
mapping = [
[name: 'apiKey', value: config.orgToken, warnIfPresent: true],
[name: 'productName', value: config.productName],
[name: 'productToken', value: config.productToken, omitIfPresent: 'projectToken'],
[name: 'userKey', value: config.userKey, warnIfPresent: true]
]
parsingClosure = { fileReadPath -> return script.readJSON (file: fileReadPath) }
serializationClosure = { configuration -> return new JsonUtils().getPrettyJsonString(configuration) }
break
case 'pip':
mapping = [
[name: "'org_token'", value: "\'${config.orgToken}\'", warnIfPresent: true],
[name: "'product_name'", value: "\'${config.productName}\'"],
[name: "'product_token'", value: "\'${config.productToken}\'"],
[name: "'user_key'", value: "\'${config.userKey}\'", warnIfPresent: true]
]
parsingClosure = { fileReadPath -> return readPythonConfig (script, fileReadPath) }
serializationClosure = { configuration -> serializePythonConfig(configuration) }
targetFile = "${inputFile}.${suffix}.py"
break
case 'sbt':
mapping = [
[name: "whitesourceOrgToken in ThisBuild", value: "\"${config.orgToken}\"", warnIfPresent: true],
[name: "whitesourceProduct in ThisBuild", value: "\"${config.productName}\""],
[name: "whitesourceServiceUrl in ThisBuild", value: "uri(\"${config.agentUrl}\")"]
// actually not supported [name: "whitesourceUserKey in ThisBuild", value: config.userKey]
]
parsingClosure = { fileReadPath -> return readScalaConfig (script, mapping, fileReadPath) }
serializationClosure = { configuration -> serializeScalaConfig (configuration) }
targetFile = inputFile
break
}
rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure)
}
static private def rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) {
def inputFilePath = "${path}${inputFile}"
def outputFilePath = "${path}${targetFile}"
@ -148,7 +90,7 @@ class WhitesourceConfigurationHelper implements Serializable {
mapping.each {
entry ->
//if (entry.warnIfPresent && moduleSpecificFile[entry.name])
if (entry.warnIfPresent && moduleSpecificFile[entry.name])
//Notify.warning(script, "Obsolete configuration ${entry.name} detected, please omit its use and rely on configuration via Piper.", 'WhitesourceConfigurationHelper')
def dependentValue = entry.omitIfPresent ? moduleSpecificFile[entry.omitIfPresent] : null
if ((entry.omitIfPresent && !dependentValue || !entry.omitIfPresent) && entry.value && entry.value != 'null' && entry.value != '')
@ -172,76 +114,6 @@ class WhitesourceConfigurationHelper implements Serializable {
config.configFilePath = outputFilePath
}
static private def readPythonConfig(script, filePath) {
def contents = script.readFile file: filePath
def lines = contents.split('\n')
def resultMap = [:]
lines.each {
line ->
List parts = line?.replaceAll(',$', '')?.split(':')
def key = parts[0]?.trim()
parts.removeAt(0)
resultMap[key] = parts.size() > 0 ? (parts as String[]).join(':').trim() : null
}
return resultMap
}
static private def serializePythonConfig(configuration) {
StringBuilder result = new StringBuilder()
configuration.each {
entry ->
if(entry.key != '}')
result.append(entry.value ? ' ' : '').append(entry.key).append(entry.value ? ': ' : '').append(entry.value ?: '').append(entry.value ? ',' : '').append('\r\n')
}
return result.toString().replaceAll(',$', '\r\n}')
}
static private def readScalaConfig(script, mapping, filePath) {
def contents = script.readFile file: filePath
def lines = contents.split('\n')
def resultMap = [:]
resultMap[SCALA_CONTENT_KEY] = []
def keys = mapping.collect( { it.name } )
lines.each {
line ->
def parts = line?.split(':=').toList()
def key = parts[0]?.trim()
if (keys.contains(key)) {
resultMap[key] = parts[1]?.trim()
} else if (line != null) {
resultMap[SCALA_CONTENT_KEY].add(line)
}
}
return resultMap
}
static private def serializeScalaConfig(configuration) {
StringBuilder result = new StringBuilder()
// write the general content
configuration[SCALA_CONTENT_KEY].each {
line ->
result.append(line)
result.append('\r\n')
}
// write the mappings
def confKeys = configuration.keySet()
confKeys.remove(SCALA_CONTENT_KEY)
confKeys.each {
key ->
def value = configuration[key]
result.append(key)
if (value != null) {
result.append(' := ').append(value)
}
result.append('\r\n')
}
return result.toString()
}
@NonCPS
static private def serializeUAConfig(configuration) {
Properties p = new Properties()

View File

@ -25,6 +25,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest {
private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this)
private JenkinsWriteFileRule writeFileRule = new JenkinsWriteFileRule(this)
private JenkinsStepRule stepRule = new JenkinsStepRule(this)
private JenkinsErrorRule errorRule = new JenkinsErrorRule(this)
@Rule
public RuleChain ruleChain = Rules
@ -36,6 +37,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest {
.around(loggingRule)
.around(writeFileRule)
.around(stepRule)
.around(errorRule)
def whitesourceOrgAdminRepositoryStub
def whitesourceStub
@ -746,6 +748,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest {
whitesourceRepositoryStub : whitesourceStub,
whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub,
descriptorUtilsStub : descriptorUtilsStub,
cvssSeverityLimit : 7,
orgToken : 'testOrgToken',
productName : 'SHC - Piper',
projectNames : [ 'piper-demo - 0.0.1' ]
@ -823,4 +826,136 @@ class WhitesourceExecuteScanTest extends BasePiperTest {
assertThat(loggingRule.log, containsString('No Open Source Software Security vulnerabilities detected.'))
assertThat(writeFileRule.files['piper_whitesource_vulnerability_report.json'], not(isEmptyOrNullString()))
}
@Test
void testCheckStatus_0() {
def error = false
try {
stepRule.step.checkStatus(0, [licensingVulnerabilities: true])
} catch (e) {
error = true
}
assertThat(error, is(false))
}
@Test
void testCheckStatus_255() {
def error = false
try {
stepRule.step.checkStatus(255, [licensingVulnerabilities: true])
} catch (e) {
error = true
assertThat(e.getMessage(), is("[whitesourceExecuteScan] The scan resulted in an error"))
}
assertThat(error, is(true))
}
@Test
void testCheckStatus_254() {
def error = false
try {
stepRule.step.checkStatus(254, [licensingVulnerabilities: true])
} catch (e) {
error = true
assertThat(e.getMessage(), is("[whitesourceExecuteScan] Whitesource found one or multiple policy violations"))
}
assertThat(error, is(true))
}
@Test
void testCheckStatus_253() {
def error = false
try {
stepRule.step.checkStatus(253, [licensingVulnerabilities: true])
} catch (e) {
error = true
assertThat(e.getMessage(), is("[whitesourceExecuteScan] The local scan client failed to execute the scan"))
}
assertThat(error, is(true))
}
@Test
void testCheckStatus_252() {
def error = false
try {
stepRule.step.checkStatus(252, [licensingVulnerabilities: true])
} catch (e) {
error = true
assertThat(e.getMessage(), is("[whitesourceExecuteScan] There was a failure in the connection to the WhiteSource servers"))
}
assertThat(error, is(true))
}
@Test
void testCheckStatus_251() {
def error = false
try {
stepRule.step.checkStatus(251, [licensingVulnerabilities: true])
} catch (e) {
error = true
assertThat(e.getMessage(), is("[whitesourceExecuteScan] The server failed to analyze the scan"))
}
assertThat(error, is(true))
}
@Test
void testCheckStatus_250() {
def error = false
try {
stepRule.step.checkStatus(250, [licensingVulnerabilities: true])
} catch (e) {
error = true
assertThat(e.getMessage(), is("[whitesourceExecuteScan] Pre-step failure"))
}
assertThat(error, is(true))
}
@Test
void testCheckStatus_127() {
def error = false
try {
stepRule.step.checkStatus(127, [licensingVulnerabilities: true])
} catch (e) {
error = true
assertThat(e.getMessage(), is("[whitesourceExecuteScan] Whitesource scan failed with unknown error code '127'"))
}
assertThat(error, is(true))
}
@Test
void testCheckStatus_vulnerability() {
def error = false
try {
stepRule.step.checkStatus(0, [licensingVulnerabilities: false, securityVulnerabilities: true, severeVulnerabilities: 5])
} catch (e) {
error = true
assertThat(e.getMessage(), is("[whitesourceExecuteScan] 5 Open Source Software Security vulnerabilities with CVSS score greater or equal 7.0 detected. - "))
}
assertThat(error, is(true))
}
@Test
void testCheckViolationStatus_0() {
def error = false
try {
stepRule.step.checkViolationStatus(0)
} catch (e) {
error = true
}
assertThat(error, is(false))
assertThat(loggingRule.log, containsString("****\r\n[whitesourceExecuteScan] No policy violations found. You can deploy to production, and set the \"Intellectual Property (IP) Scan Plan\" in Sirius to completed. \r\n****"))
}
@Test
void testCheckViolationStatus_5() {
def error = false
try {
stepRule.step.checkViolationStatus(5)
} catch (e) {
error = true
assertThat(e.getMessage(), is("[whitesourceExecuteScan] Whitesource found 5 policy violations for your product"))
}
assertThat(error, is(true))
}
}

View File

@ -9,7 +9,7 @@ import util.JenkinsReadFileRule
import util.JenkinsWriteFileRule
import util.Rules
import static org.hamcrest.Matchers.*
import static org.hamcrest.Matchers.containsString
import static org.junit.Assert.assertThat
class WhiteSourceConfigurationHelperTest extends BasePiperTest {
@ -22,84 +22,30 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest {
.around(jrfr)
.around(jwfr)
private static getMapping() {
return [
[name: "whitesourceOrgToken in ThisBuild", value: "config.orgToken", warnIfPresent: true],
[name: "whitesourceProduct in ThisBuild", value: "config.whitesourceProductName"]
]
}
@Before
void init() {
helper.registerAllowedMethod('readJSON', [Map], {return [:]})
helper.registerAllowedMethod('readProperties', [Map], {return new Properties()})
}
@Test
void testReadScalaConfig() {
def resMap = WhitesourceConfigurationHelper.readScalaConfig(nullScript, getMapping(), "build.sbt")
assertThat(resMap, hasKey(WhitesourceConfigurationHelper.SCALA_CONTENT_KEY))
// mapping tokens should be removed from parsed content
assertThat(resMap[WhitesourceConfigurationHelper.SCALA_CONTENT_KEY], not(hasItem(containsString("whitesourceOrgToken in ThisBuild"))))
assertThat(resMap, hasEntry("whitesourceOrgToken in ThisBuild", "\"org-token\""))
assertThat(resMap, hasEntry("whitesourceProduct in ThisBuild", "\"PRODUCT VERSION\""))
}
@Test
void testSerializeScalaConfig() {
def resMap = [
"whitesourceOrgToken in ThisBuild": "\"some-long-hash-value\"",
"whitesourceProduct in ThisBuild": "\"PRODUCT IDENTIFIER\"",
"whitesourceServiceUrl in ThisBuild": "uri(\"http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp\")"
]
resMap[WhitesourceConfigurationHelper.SCALA_CONTENT_KEY] = ["// build.sbt -- scala build file", "name := \"minimal-scala\"", "libraryDependencies += \"org.scalatest\" %% \"scalatest\" % \"2.2.4\" % \"test\""]
def fileContent = WhitesourceConfigurationHelper.serializeScalaConfig(resMap)
resMap[WhitesourceConfigurationHelper.SCALA_CONTENT_KEY].each {
line ->
assertThat(fileContent, containsString("${line}\r"))
}
assertThat(fileContent, containsString("whitesourceOrgToken in ThisBuild := \"some-long-hash-value\""))
assertThat(fileContent, containsString("whitesourceProduct in ThisBuild := \"PRODUCT IDENTIFIER\""))
assertThat(fileContent, containsString("whitesourceServiceUrl in ThisBuild := uri(\"http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp\")"))
}
@Test
void testExtendConfigurationFileUnifiedAgent() {
WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'unifiedAgent', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./")
void testExtendConfigurationFileUnifiedAgentPip() {
WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./")
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("apiKey=abcd"))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productName=name"))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productToken=1234"))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("userKey=0000"))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("python.resolveDependencies=true"))
}
@Test
void testExtendConfigurationFileNpm() {
WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'npm', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./")
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"apiKey\": \"abcd\","))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"productName\": \"name\","))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"productToken\": \"1234\","))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"userKey\": \"0000\""))
}
@Test
void testExtendConfigurationFilePip() {
WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './setup.py', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./")
assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'org_token': 'abcd',"))
assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'product_name': 'name',"))
assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'product_token': '1234',"))
assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'user_key': '0000'"))
}
@Test
void testExtendConfigurationFileSbt() {
WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'sbt', configFilePath: './build.sbt', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000', agentUrl: 'http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp'], "./")
assertThat(jwfr.files['./build.sbt'], containsString("whitesourceOrgToken in ThisBuild := \"abcd\""))
assertThat(jwfr.files['./build.sbt'], containsString("whitesourceProduct in ThisBuild := \"name\""))
assertThat(jwfr.files['./build.sbt'], containsString("whitesourceServiceUrl in ThisBuild := uri(\"http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp\")"))
void testExtendConfigurationFileUnifiedAgentVerbose() {
WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', verbose: true, configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./")
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("apiKey=abcd"))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productName=name"))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productToken=1234"))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("userKey=0000"))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("python.resolveDependencies=true"))
assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("log.level=debug"))
}
}

View File

@ -14,6 +14,19 @@ import static com.sap.piper.Prerequisites.checkScript
@Field String STEP_NAME = 'whitesourceExecuteScan'
@Field Set GENERAL_CONFIG_KEYS = [
'orgAdminUserTokenCredentialsId',
'orgToken',
'productName',
'productVersion',
'productToken',
'projectNames',
'scanType',
'serviceUrl',
'internalServiceUrl',
'userTokenCredentialsId',
'verbose'
]
@Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS + [
'agentDownloadUrl',
'agentFileName',
'agentParameters',
@ -23,29 +36,19 @@ import static com.sap.piper.Prerequisites.checkScript
'configFilePath',
'dockerImage',
'dockerWorkspace',
'internalServiceUrl',
'jreDownloadUrl',
'licensingVulnerabilities',
'orgAdminUserTokenCredentialsId',
'orgToken',
'parallelLimit',
'reporting',
'scanType',
'securityVulnerabilities',
'cvssSeverityLimit',
'stashContent',
'timeout',
'productName',
'productVersion',
'productToken',
'projectNames',
'serviceUrl',
'userTokenCredentialsId',
'vulnerabilityReportFileName',
'vulnerabilityReportTitle',
'whitesourceAccessor',
'verbose'
'whitesourceAccessor'
]
@Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS
@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS
void call(Map parameters = [:]) {
@ -77,6 +80,7 @@ void call(Map parameters = [:]) {
.withMandatoryProperty('productName')
.use()
config.cvssSeverityLimit = Integer.valueOf(config.cvssSeverityLimit)
config.stashContent = utils.unstashAll(config.stashContent)
config.projectNames = (config.projectNames instanceof List) ? config.projectNames : config.projectNames?.tokenize(',')
parameters.projectNames = config.projectNames
@ -199,7 +203,6 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU
if (config.jreDownloadUrl) {
sh "rm -r ./bin ./conf ./legal ./lib ./man"
javaCmd = './bin/java'
}
// archive whitesource result files
@ -228,8 +231,10 @@ void analyseWhitesourceResults(Map config, WhitesourceRepository repository, Whi
archiveArtifacts artifacts: pdfName
echo "A summary of the Whitesource findings was stored as artifact under the name ${pdfName}"
int violationCount = fetchViolationCount(config, repository)
checkViolationStatus(violationCount)
if(config.licensingVulnerabilities) {
def violationCount = fetchViolationCount(config, repository)
checkViolationStatus(violationCount)
}
if (config.securityVulnerabilities)
config.severeVulnerabilities = checkSecurityViolations(config, repository)
@ -255,7 +260,7 @@ void checkViolationStatus(int violationCount) {
if (violationCount == 0) {
echo "****\r\n[${STEP_NAME}] No policy violations found. You can deploy to production, and set the \"Intellectual Property (IP) Scan Plan\" in Sirius to completed. \r\n****"
} else {
error "[${STEP_NAME}] Whitesource found $violationCount policy violations for your product"
error "[${STEP_NAME}] Whitesource found ${violationCount} policy violations for your product"
}
}
@ -265,7 +270,7 @@ int checkSecurityViolations(Map config, WhitesourceRepository repository) {
def severeVulnerabilities = 0
whitesourceVulnerabilities.each {
item ->
if (item.vulnerability.score >= 7 || item.vulnerability.cvss3_score >= 7)
if ((item.vulnerability.score >= config.cvssSeverityLimit || item.vulnerability.cvss3_score >= config.cvssSeverityLimit) && config.cvssSeverityLimit >= 0)
severeVulnerabilities++
}
@ -305,6 +310,9 @@ void checkStatus(int statusCode, config) {
case 251:
errorMessage += "The server failed to analyze the scan"
break
case 250:
errorMessage += "Pre-step failure"
break
default:
errorMessage += "Whitesource scan failed with unknown error code '${statusCode}'"
}